From a5f0e022c60eb9e9cc8d24d66254ba3a9550e95f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 22 Jul 2020 11:24:36 +0200 Subject: [PATCH] Start changing the format to work in the Nix sandbox --- fetch/fetch.go | 51 +++++++++++++++++++++------------- fetch/sum.go | 9 +----- formats/gomod2nix/gomod2nix.go | 28 +++++++++++++------ types/types.go | 2 ++ 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/fetch/fetch.go b/fetch/fetch.go index de265f1..21dd80e 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -8,9 +8,11 @@ import ( "github.com/tweag/gomod2nix/formats/gomod2nix" "github.com/tweag/gomod2nix/types" "golang.org/x/mod/modfile" + "golang.org/x/mod/module" "golang.org/x/tools/go/vcs" "io/ioutil" "os/exec" + "regexp" "sort" "strings" ) @@ -18,7 +20,7 @@ import ( type packageJob struct { importPath string goPackagePath string - rev string + sumVersion string } type packageResult struct { @@ -35,7 +37,7 @@ func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob, "goPackagePath": j.goPackagePath, }).Info("Worker received job") - pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.rev) + pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.sumVersion) results <- &packageResult{ err: err, pkg: pkg, @@ -43,11 +45,6 @@ func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob, } } -// It's a relatively common idiom to tag storage/v1.0.0 -func mkNewRev(goPackagePath string, repoRoot *vcs.RepoRoot, rev string) string { - return fmt.Sprintf("%s/%s", strings.TrimPrefix(goPackagePath, repoRoot.Root+"/"), rev) -} - func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, depsNixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) { log.WithFields(log.Fields{ @@ -92,12 +89,12 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep "sumPath": goSumPath, }).Info("Parsing go.sum") - revs, err := parseGoSum(goSumPath) + sumVersions, err := parseGoSum(goSumPath) if err != nil { return nil, err } - numJobs := len(revs) + numJobs := len(sumVersions) if numJobs < numWorkers { numWorkers = numJobs } @@ -114,17 +111,18 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep log.WithFields(log.Fields{ "numJobs": numJobs, }).Info("Queuing jobs") - for goPackagePath, rev := range revs { + for goPackagePath, sumVersion := range sumVersions { // Check for replacement path (only original goPackagePath is recorded in go.sum) importPath := goPackagePath v, ok := replace[goPackagePath] if ok { importPath = v } + jobs <- &packageJob{ importPath: importPath, goPackagePath: goPackagePath, - rev: rev, + sumVersion: sumVersion, } } close(jobs) @@ -157,13 +155,27 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep return pkgs, nil } -func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, rev string) (*types.Package, error) { +func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) { repoRoot, err := vcs.RepoRootForImportPath(importPath, false) if err != nil { return nil, err } - newRev := mkNewRev(goPackagePath, repoRoot, rev) + commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z]*?[0-9]{14}-(.*?)$`) + rev := strings.TrimSuffix(sumVersion, "+incompatible") + if commitShaRev.MatchString(rev) { + rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1] + } + + goPackagePathPrefix, _, _ := module.SplitPathVersion(goPackagePath) + + // Relative path within the repo + relPath := strings.TrimPrefix(goPackagePathPrefix, repoRoot.Root+"/") + if relPath == goPackagePathPrefix { + relPath = "" + } + + newRev := fmt.Sprintf("%s/%s", relPath, rev) if len(caches) > 0 { log.WithFields(log.Fields{ @@ -242,13 +254,12 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka return &types.Package{ GoPackagePath: goPackagePath, URL: repoRoot.Repo, - // It may feel appealing to use output.Rev to get the full git hash - // However, this has the major downside of not being able to be checked against an - // older output file (as the revs) don't match - // - // This is used to skip fetching where the previous package path & rev are still the same - Rev: rev, - Sha256: output.Sha256, + Rev: output.Rev, + Sha256: output.Sha256, + // This is used to skip fetching where the previous package path & versions are still the same + // It's also used to construct the vendor directory in the Nix build + SumVersion: sumVersion, + RelPath: relPath, }, nil } diff --git a/fetch/sum.go b/fetch/sum.go index def91e7..7386d1b 100644 --- a/fetch/sum.go +++ b/fetch/sum.go @@ -4,12 +4,10 @@ import ( "bytes" "fmt" "io/ioutil" - "regexp" "strings" ) func parseGoSum(file string) (map[string]string, error) { - commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z]*?[0-9]{14}-(.*?)$`) // Read go.mod data, err := ioutil.ReadFile(file) @@ -28,12 +26,7 @@ func parseGoSum(file string) (map[string]string, error) { return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) } - rev := strings.TrimSuffix(strings.TrimSuffix(f[1], "/go.mod"), "+incompatible") - if commitShaRev.MatchString(rev) { - rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1] - } - - pkgs[f[0]] = rev + pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod") } return pkgs, nil diff --git a/formats/gomod2nix/gomod2nix.go b/formats/gomod2nix/gomod2nix.go index a28e9b4..f48fb71 100644 --- a/formats/gomod2nix/gomod2nix.go +++ b/formats/gomod2nix/gomod2nix.go @@ -8,22 +8,32 @@ import ( "io/ioutil" ) -type packageT struct { +type fetchInfo struct { Type string `toml:"type"` URL string `toml:"url"` Rev string `toml:"rev"` Sha256 string `toml:"sha256"` } +type packageT struct { + SumVersion string `toml:"sumVersion"` + RelPath string `toml:"relPath"` + Fetch *fetchInfo `toml:"fetch"` +} + func Marshal(pkgs []*types.Package) ([]byte, error) { result := make(map[string]*packageT) for _, pkg := range pkgs { result[pkg.GoPackagePath] = &packageT{ - Type: "git", - URL: pkg.URL, - Rev: pkg.Rev, - Sha256: pkg.Sha256, + SumVersion: pkg.SumVersion, + RelPath: pkg.RelPath, + Fetch: &fetchInfo{ + Type: "git", + URL: pkg.URL, + Rev: pkg.Rev, + Sha256: pkg.Sha256, + }, } } @@ -60,9 +70,11 @@ func LoadGomod2Nix(filePath string) map[string]*types.Package { for k, v := range result { ret[k] = &types.Package{ GoPackagePath: k, - URL: v.URL, - Rev: v.Rev, - Sha256: v.Sha256, + URL: v.Fetch.URL, + Rev: v.Fetch.Rev, + Sha256: v.Fetch.Sha256, + SumVersion: v.SumVersion, + RelPath: v.RelPath, } } diff --git a/types/types.go b/types/types.go index b628bdc..a844b66 100644 --- a/types/types.go +++ b/types/types.go @@ -5,4 +5,6 @@ type Package struct { URL string Rev string Sha256 string + SumVersion string + RelPath string }