Most basic functionality works

fetch.go

@@ -3,8 +3,11 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"golang.org/x/mod/modfile"
 	"golang.org/x/tools/go/vcs"
+	"io/ioutil"
 	"os/exec"
+	"sort"
 )
 
 type Package struct {
@@ -14,16 +17,87 @@ type Package struct {
 	Hash          string
 }
 
-func fetchPackage(replace map[string]string, goPackagePath string, rev string) (*Package, error) {
+func FetchPackages(goModPath string, goSumPath string, numWorkers int, keepGoing bool) ([]*Package, error) {
 
-	// Check for replacement path (only original goPackagePath is recorded in go.sum)
-	repo := goPackagePath
-	v, ok := replace[goPackagePath]
-	if ok {
-		repo = v
+	// Read go.mod
+	data, err := ioutil.ReadFile(goModPath)
+	if err != nil {
+		return nil, err
 	}
 
-	repoRoot, err := vcs.RepoRootForImportPath(repo, false)
+	// Parse go.mod
+	mod, err := modfile.Parse(goModPath, data, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// // Parse require
+	// require := make(map[string]module.Version)
+	// for _, req := range mod.Require {
+	// 	require[req.Mod.Path] = req.Mod
+	// }
+
+	// Map repos -> replacement repo
+	replace := make(map[string]string)
+	for _, repl := range mod.Replace {
+		replace[repl.Old.Path] = repl.New.Path
+	}
+
+	revs, err := parseGoSum(goSumPath)
+	if err != nil {
+		return nil, err
+	}
+
+	numJobs := len(revs)
+	if numJobs < numWorkers {
+		numWorkers = numJobs
+	}
+
+	jobs := make(chan *packageJob, numJobs)
+	results := make(chan *packageResult, numJobs)
+	for i := 0; i <= numWorkers; i++ {
+		go worker(i, replace, jobs, results)
+	}
+
+	for goPackagePath, rev := range revs {
+		// Check for replacement path (only original goPackagePath is recorded in go.sum)
+		importPath := goPackagePath
+		v, ok := replace[goPackagePath]
+		if ok {
+			importPath = v
+		}
+		jobs <- &packageJob{
+			importPath:    importPath,
+			goPackagePath: goPackagePath,
+			rev:           rev,
+		}
+	}
+	close(jobs)
+
+	var pkgs []*Package
+	for i := 1; i <= numJobs; i++ {
+		result := <-results
+		if result.err != nil {
+			if keepGoing {
+				fmt.Println(result.err)
+				continue
+			} else {
+				return nil, result.err
+			}
+		}
+
+		pkgs = append(pkgs, result.pkg)
+	}
+
+	sort.Slice(pkgs, func(i, j int) bool {
+		return pkgs[i].GoPackagePath < pkgs[j].GoPackagePath
+	})
+
+	return pkgs, nil
+}
+
+func fetchPackage(importPath string, goPackagePath string, rev string) (*Package, error) {
+	repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
 	if err != nil {
 		return nil, err
 	}

main.go

@@ -3,12 +3,11 @@ package main // import "github.com/tweag/gomod2nix"
 import (
 	"flag"
 	"fmt"
-	"golang.org/x/mod/modfile"
-	"io/ioutil"
 	"path/filepath"
 )
 
 type packageJob struct {
+	importPath    string
 	goPackagePath string
 	rev           string
 }
@@ -20,7 +19,7 @@ type packageResult struct {
 
 func worker(id int, replace map[string]string, jobs <-chan *packageJob, results chan<- *packageResult) {
 	for j := range jobs {
-		pkg, err := fetchPackage(replace, j.goPackagePath, j.rev)
+		pkg, err := fetchPackage(j.importPath, j.goPackagePath, j.rev)
 		results <- &packageResult{
 			err: err,
 			pkg: pkg,
@@ -30,65 +29,19 @@ func worker(id int, replace map[string]string, jobs <-chan *packageJob, results
 
 func main() {
 
-	// var jobs = flag.Int("jobs", 20, "Number of parallel jobs")
 	flag.Parse()
 
-	numWorkers := 10
-
-	// directory := "testdata"
+	numWorkers := 20
+	keepGoing := true
 	directory := "./"
 
-	// Read go.mod
-	data, err := ioutil.ReadFile(filepath.Join(directory, "go.mod"))
+	pkgs, err := FetchPackages(filepath.Join(directory, "go.mod"), filepath.Join(directory, "go.sum"), numWorkers, keepGoing)
 	if err != nil {
 		panic(err)
 	}
 
-	// Parse go.mod
-	mod, err := modfile.Parse("go.mod", data, nil)
-	if err != nil {
-		panic(err)
-	}
-
-	// // Parse require
-	// require := make(map[string]module.Version)
-	// for _, req := range mod.Require {
-	// 	require[req.Mod.Path] = req.Mod
-	// }
-
-	// Map repos -> replacement repo
-	replace := make(map[string]string)
-	for _, repl := range mod.Replace {
-		replace[repl.Old.Path] = repl.New.Path
-	}
-
-	revs, err := parseGoSum(filepath.Join(directory, "go.sum"))
-	if err != nil {
-		panic(err)
-	}
-
-	numJobs := len(revs)
-	if numJobs < numWorkers {
-		numWorkers = numJobs
-	}
-
-	jobs := make(chan *packageJob, numJobs)
-	results := make(chan *packageResult, numJobs)
-	for i := 0; i <= numWorkers; i++ {
-		go worker(i, replace, jobs, results)
-	}
-
-	for goPackagePath, rev := range revs {
-		jobs <- &packageJob{
-			goPackagePath: goPackagePath,
-			rev:           rev,
-		}
-	}
-	close(jobs)
-
-	for i := 1; i <= numJobs; i++ {
-		result := <-results
-		fmt.Println(result)
+	for _, pkg := range pkgs {
+		fmt.Println(pkg)
 	}
 
 }