forked from mirrors/gomod2nix
Reintroduce the max workers flag
It turns out that this is actually useful to limit the number of open files. Fixes https://github.com/tweag/gomod2nix/issues/50.
This commit is contained in:
parent
bf3e9afe32
commit
e1d46e6973
3 changed files with 20 additions and 4 deletions
|
@ -30,7 +30,7 @@ type goModDownload struct {
|
|||
GoModSum string
|
||||
}
|
||||
|
||||
func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, error) {
|
||||
func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) {
|
||||
goModPath := filepath.Join(directory, "go.mod")
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
|
@ -81,7 +81,7 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
|
|||
log.Info("Done downloading dependencies")
|
||||
}
|
||||
|
||||
executor := lib.NewParallellExecutor()
|
||||
executor := lib.NewParallellExecutor(numWorkers)
|
||||
var mux sync.Mutex
|
||||
|
||||
cache := schema.ReadCache(goMod2NixPath)
|
||||
|
@ -108,6 +108,10 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
|
|||
}
|
||||
|
||||
executor.Add(func() error {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Calculating NAR hash")
|
||||
|
||||
h := sha256.New()
|
||||
err := nar.DumpPath(h, dl.Dir)
|
||||
if err != nil {
|
||||
|
@ -126,6 +130,10 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
|
|||
|
||||
addPkg(pkg)
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Done calculating NAR hash")
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
|
|
@ -9,17 +9,19 @@ type ParallellExecutor struct {
|
|||
errChan chan error
|
||||
wg *sync.WaitGroup
|
||||
mux *sync.Mutex
|
||||
guard chan struct{}
|
||||
|
||||
// Error returned by Wait(), cached for other Wait() invocations
|
||||
err error
|
||||
done bool
|
||||
}
|
||||
|
||||
func NewParallellExecutor() *ParallellExecutor {
|
||||
func NewParallellExecutor(maxWorkers int) *ParallellExecutor {
|
||||
return &ParallellExecutor{
|
||||
errChan: make(chan error),
|
||||
mux: new(sync.Mutex),
|
||||
wg: new(sync.WaitGroup),
|
||||
guard: make(chan struct{}, maxWorkers),
|
||||
|
||||
err: nil,
|
||||
done: false,
|
||||
|
@ -29,8 +31,13 @@ func NewParallellExecutor() *ParallellExecutor {
|
|||
func (e *ParallellExecutor) Add(fn func() error) {
|
||||
e.wg.Add(1)
|
||||
|
||||
e.guard <- struct{}{} // Block until a worker is available
|
||||
|
||||
go func() {
|
||||
defer e.wg.Done()
|
||||
defer func() {
|
||||
<-e.guard
|
||||
}()
|
||||
|
||||
err := fn()
|
||||
if err != nil {
|
||||
|
|
3
main.go
3
main.go
|
@ -13,6 +13,7 @@ import (
|
|||
func main() {
|
||||
|
||||
var directory = flag.String("dir", "./", "Go project directory")
|
||||
var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
|
||||
var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
|
||||
flag.Parse()
|
||||
|
||||
|
@ -23,7 +24,7 @@ func main() {
|
|||
|
||||
goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
|
||||
outFile := goMod2NixPath
|
||||
pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath)
|
||||
pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath, *maxJobs)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue