Reintroduce the max workers flag

It turns out that this is actually useful to limit the number of open files.

Fixes https://github.com/tweag/gomod2nix/issues/50.
This commit is contained in:
adisbladis 2022-05-31 00:08:36 +08:00
parent bf3e9afe32
commit e1d46e6973
3 changed files with 20 additions and 4 deletions

View file

@ -30,7 +30,7 @@ type goModDownload struct {
GoModSum string
}
func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, error) {
func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) {
goModPath := filepath.Join(directory, "go.mod")
log.WithFields(log.Fields{
@ -81,7 +81,7 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
log.Info("Done downloading dependencies")
}
executor := lib.NewParallellExecutor()
executor := lib.NewParallellExecutor(numWorkers)
var mux sync.Mutex
cache := schema.ReadCache(goMod2NixPath)
@ -108,6 +108,10 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
}
executor.Add(func() error {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Info("Calculating NAR hash")
h := sha256.New()
err := nar.DumpPath(h, dl.Dir)
if err != nil {
@ -126,6 +130,10 @@ func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, er
addPkg(pkg)
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Info("Done calculating NAR hash")
return nil
})
}

View file

@ -9,17 +9,19 @@ type ParallellExecutor struct {
errChan chan error
wg *sync.WaitGroup
mux *sync.Mutex
guard chan struct{}
// Error returned by Wait(), cached for other Wait() invocations
err error
done bool
}
func NewParallellExecutor() *ParallellExecutor {
func NewParallellExecutor(maxWorkers int) *ParallellExecutor {
return &ParallellExecutor{
errChan: make(chan error),
mux: new(sync.Mutex),
wg: new(sync.WaitGroup),
guard: make(chan struct{}, maxWorkers),
err: nil,
done: false,
@ -29,8 +31,13 @@ func NewParallellExecutor() *ParallellExecutor {
func (e *ParallellExecutor) Add(fn func() error) {
e.wg.Add(1)
e.guard <- struct{}{} // Block until a worker is available
go func() {
defer e.wg.Done()
defer func() {
<-e.guard
}()
err := fn()
if err != nil {

View file

@ -13,6 +13,7 @@ import (
func main() {
var directory = flag.String("dir", "./", "Go project directory")
var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
flag.Parse()
@ -23,7 +24,7 @@ func main() {
goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
outFile := goMod2NixPath
pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath)
pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath, *maxJobs)
if err != nil {
panic(err)
}