diff --git a/fetch/fetch.go b/fetch/fetch.go index a6d2501..2da7463 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -3,6 +3,7 @@ package fetch import ( "encoding/json" "fmt" + "github.com/tweag/gomod2nix/formats/gomod2nix" "github.com/tweag/gomod2nix/types" "golang.org/x/mod/modfile" "golang.org/x/tools/go/vcs" @@ -23,9 +24,9 @@ type packageResult struct { err error } -func worker(id int, replace map[string]string, jobs <-chan *packageJob, results chan<- *packageResult) { +func worker(id int, cache map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) { for j := range jobs { - pkg, err := fetchPackage(j.importPath, j.goPackagePath, j.rev) + pkg, err := fetchPackage(cache, j.importPath, j.goPackagePath, j.rev) results <- &packageResult{ err: err, pkg: pkg, @@ -33,7 +34,12 @@ func worker(id int, replace map[string]string, jobs <-chan *packageJob, results } } -func FetchPackages(goModPath string, goSumPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) { +// It's a relatively common idiom to tag storage/v1.0.0 +func mkNewRev(goPackagePath string, repoRoot *vcs.RepoRoot, rev string) string { + return fmt.Sprintf("%s/%s", strings.TrimPrefix(goPackagePath, repoRoot.Root+"/"), rev) +} + +func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) { // Read go.mod data, err := ioutil.ReadFile(goModPath) @@ -47,6 +53,8 @@ func FetchPackages(goModPath string, goSumPath string, numWorkers int, keepGoing return nil, err } + cache := gomod2nix.LoadGomod2Nix(goMod2NixPath) + // // Parse require // require := make(map[string]module.Version) // for _, req := range mod.Require { @@ -72,7 +80,7 @@ func FetchPackages(goModPath string, goSumPath string, numWorkers int, keepGoing jobs := make(chan *packageJob, numJobs) results := make(chan *packageResult, numJobs) for i := 0; i <= numWorkers; i++ { - go worker(i, replace, jobs, results) + go worker(i, cache, jobs, results) } for goPackagePath, rev := range revs { @@ -112,12 +120,22 @@ func FetchPackages(goModPath string, goSumPath string, numWorkers int, keepGoing return pkgs, nil } -func fetchPackage(importPath string, goPackagePath string, rev string) (*types.Package, error) { +func fetchPackage(cache map[string]*types.Package, importPath string, goPackagePath string, rev string) (*types.Package, error) { repoRoot, err := vcs.RepoRootForImportPath(importPath, false) if err != nil { return nil, err } + newRev := mkNewRev(goPackagePath, repoRoot, rev) + cached, ok := cache[goPackagePath] + if ok { + for _, rev := range []string{rev, newRev} { + if cached.Rev == rev { + return cached, nil + } + } + } + if repoRoot.VCS.Name != "Git" { return nil, fmt.Errorf("Only git repositories are supported") } @@ -139,10 +157,7 @@ func fetchPackage(importPath string, goPackagePath string, rev string) (*types.P "--url", repoRoot.Repo, "--rev", rev).Output() if err != nil { - // It's a relatively common idiom to tag storage/v1.0.0 - newRev := fmt.Sprintf("%s/%s", strings.TrimPrefix(goPackagePath, repoRoot.Root+"/"), rev) originalErr := err - stdout, err = exec.Command( "nix-prefetch-git", "--quiet", diff --git a/formats/gomod2nix/gomod2nix.go b/formats/gomod2nix/gomod2nix.go index 8fffdd4..eda708f 100644 --- a/formats/gomod2nix/gomod2nix.go +++ b/formats/gomod2nix/gomod2nix.go @@ -53,5 +53,14 @@ func LoadGomod2Nix(filePath string) map[string]*types.Package { return ret } + for k, v := range result { + ret[k] = &types.Package{ + GoPackagePath: k, + URL: v.URL, + Rev: v.Rev, + Sha256: v.Sha256, + } + } + return ret } diff --git a/main.go b/main.go index 7b300d2..8c69628 100644 --- a/main.go +++ b/main.go @@ -14,16 +14,25 @@ func main() { flag.Parse() - numWorkers := 20 - keepGoing := true - directory := "./" + numWorkers := 1 + keepGoing := false + // directory := "./" + directory := "./testdata/vuls" outFile := "gomod2nix.toml" - pkgs, err := fetch.FetchPackages(filepath.Join(directory, "go.mod"), filepath.Join(directory, "go.sum"), numWorkers, keepGoing) + goModPath := filepath.Join(directory, "go.mod") + goSumPath := filepath.Join(directory, "go.sum") + goMod2NixPath := "./gomod2nix.toml" + + pkgs, err := fetch.FetchPackages(goModPath, goSumPath, goMod2NixPath, numWorkers, keepGoing) if err != nil { panic(err) } + if true { + panic("Success") + } + // output, err := buildgopackage.Marshal(pkgs) output, err := gomod2nix.Marshal(pkgs) if err != nil {