diff --git a/fetch/fetch.go b/fetch/fetch.go index 5457fb1..3163506 100644 --- a/fetch/fetch.go +++ b/fetch/fetch.go @@ -1,22 +1,18 @@ package fetch import ( + "bytes" "encoding/json" "fmt" + "io" "io/ioutil" - "os" "os/exec" - "path/filepath" - "regexp" - "sort" + "path" "strings" log "github.com/sirupsen/logrus" - "github.com/tweag/gomod2nix/formats/gomod2nix" "github.com/tweag/gomod2nix/types" "golang.org/x/mod/modfile" - "golang.org/x/mod/module" - "golang.org/x/tools/go/vcs" ) type packageJob struct { @@ -30,21 +26,15 @@ type packageResult struct { err error } -func worker(id int, cache map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) { - log.WithField("workerId", id).Info("Starting worker process") - - for j := range jobs { - log.WithFields(log.Fields{ - "workerId": id, - "goPackagePath": j.goPackagePath, - }).Info("Worker received job") - - pkg, err := fetchPackage(cache, j.importPath, j.goPackagePath, j.sumVersion) - results <- &packageResult{ - err: err, - pkg: pkg, - } - } +type goModDownload struct { + Path string + Version string + Info string + GoMod string + Zip string + Dir string + Sum string + GoModSum string } func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) { @@ -65,198 +55,80 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, num return nil, err } - caches := gomod2nix.LoadGomod2Nix(goMod2NixPath) - // Map repos -> replacement repo replace := make(map[string]string) for _, repl := range mod.Replace { replace[repl.New.Path] = repl.Old.Path } - log.WithFields(log.Fields{ - "sumPath": goSumPath, - }).Info("Parsing go.sum") - - sumVersions, err := parseGoSum(goSumPath) - if err != nil { - return nil, err - } - - numJobs := len(sumVersions) - if numJobs < numWorkers { - numWorkers = numJobs - } - - log.WithFields(log.Fields{ - "numWorkers": numWorkers, - }).Info("Starting worker processes") - jobs := make(chan *packageJob, numJobs) - results := make(chan *packageResult, numJobs) - for i := 0; i <= numWorkers; i++ { - go worker(i, caches, jobs, results) - } - - log.WithFields(log.Fields{ - "numJobs": numJobs, - }).Info("Queuing jobs") - for importPath, sumVersion := range sumVersions { - // Check for replacement path (only original goPackagePath is recorded in go.sum) - goPackagePath := importPath - v, ok := replace[goPackagePath] - if ok { - goPackagePath = v - } - - jobs <- &packageJob{ - importPath: importPath, - goPackagePath: goPackagePath, - sumVersion: sumVersion, - } - } - close(jobs) - - var pkgs []*types.Package - for i := 1; i <= numJobs; i++ { - result := <-results - - log.WithFields(log.Fields{ - "current": i, - "total": numJobs, - }).Info("Received finished job") - - if result.err != nil { - if keepGoing { - fmt.Println(result.err) - continue - } else { - return nil, result.err - } - } - - pkgs = append(pkgs, result.pkg) - } - - sort.Slice(pkgs, func(i, j int) bool { - return pkgs[i].GoPackagePath < pkgs[j].GoPackagePath - }) - - return pkgs, nil -} - -func fetchPackage(cache map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) { - repoRoot, err := vcs.RepoRootForImportPath(importPath, false) - if err != nil { - return nil, err - } - - commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z\-]*?[0-9]{14}-(.*?)$`) - rev := strings.TrimSuffix(sumVersion, "+incompatible") - if commitShaRev.MatchString(rev) { - rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1] - } - - importPathPrefix, pathMajor, _ := module.SplitPathVersion(importPath) - - // Relative path within the repo - relPath := strings.TrimPrefix(importPathPrefix, repoRoot.Root+"/") - if relPath == importPathPrefix { - relPath = "" - } - - log.WithFields(log.Fields{ - "goPackagePath": goPackagePath, - }).Info("Checking previous invocation cache") + var modDownloads []*goModDownload { - cached, ok := cache[goPackagePath] - if ok { - if cached.SumVersion == sumVersion { - log.WithFields(log.Fields{ - "goPackagePath": goPackagePath, - }).Info("Returning cached entry") - return cached, nil - } + log.WithFields(log.Fields{ + "sumPath": goSumPath, + }).Info("Downloading dependencies") + + stdout, err := exec.Command( + "go", "mod", "download", "--json", + ).Output() + if err != nil { + return nil, err } - } - if repoRoot.VCS.Name != "Git" { - return nil, fmt.Errorf("Only git repositories are supported") - } - - type prefetchOutput struct { - URL string `json:"url"` - Rev string `json:"rev"` - Sha256 string `json:"sha256"` - Path string `json:"path"` - // date string - // fetchSubmodules bool - // deepClone bool - // leaveDotGit bool - } - - log.WithFields(log.Fields{ - "goPackagePath": goPackagePath, - "rev": rev, - }).Info("Cache miss, fetching") - stdout, err := exec.Command( - "nix-prefetch-git", - "--quiet", - "--fetch-submodules", - "--url", repoRoot.Repo, - "--rev", rev).Output() - if err != nil { - newRev := fmt.Sprintf("%s/%s", relPath, rev) + dec := json.NewDecoder(bytes.NewReader(stdout)) + for { + var dl *goModDownload + err := dec.Decode(&dl) + if err == io.EOF { + break + } + modDownloads = append(modDownloads, dl) + } log.WithFields(log.Fields{ - "goPackagePath": goPackagePath, - "rev": newRev, - }).Info("Fetching failed, retrying with different rev format") - originalErr := err - stdout, err = exec.Command( - "nix-prefetch-git", - "--quiet", - "--fetch-submodules", - "--url", repoRoot.Repo, - "--rev", newRev).Output() + "sumPath": goSumPath, + }).Info("Done downloading dependencies") + } + + packages := []*types.Package{} + for _, dl := range modDownloads { + + goPackagePath, hasReplace := replace[dl.Path] + if !hasReplace { + goPackagePath = dl.Path + } + + var storePath string + { + stdout, err := exec.Command( + "nix", "eval", "--impure", "--expr", + fmt.Sprintf("builtins.path { name = \"%s_%s\"; path = \"%s\"; }", path.Base(goPackagePath), dl.Version, dl.Dir), + ).Output() + if err != nil { + return nil, err + } + storePath = string(stdout)[1 : len(stdout)-2] + } + + stdout, err := exec.Command( + "nix-store", "--query", "--hash", storePath, + ).Output() if err != nil { - log.WithFields(log.Fields{ - "goPackagePath": goPackagePath, - }).Error("Fetching failed") - return nil, originalErr + return nil, err + } + hash := strings.TrimSpace(string(stdout)) + + pkg := &types.Package{ + GoPackagePath: goPackagePath, + Version: dl.Version, + Hash: hash, + } + if hasReplace { + pkg.ReplacedPath = dl.Path } - rev = newRev + packages = append(packages, pkg) } - var output *prefetchOutput - err = json.Unmarshal(stdout, &output) - if err != nil { - return nil, err - } - - vendorPath := "" - if importPath != goPackagePath { - vendorPath = importPath - } - - if relPath == "" && pathMajor != "" { - p := filepath.Join(output.Path, pathMajor) - _, err := os.Stat(p) - if err == nil { - fmt.Println(pathMajor) - relPath = strings.TrimPrefix(pathMajor, "/") - } - } - - return &types.Package{ - GoPackagePath: goPackagePath, - URL: repoRoot.Repo, - Rev: output.Rev, - Sha256: output.Sha256, - // This is used to skip fetching where the previous package path & versions are still the same - // It's also used to construct the vendor directory in the Nix build - SumVersion: sumVersion, - RelPath: relPath, - VendorPath: vendorPath, - }, nil + return packages, nil } diff --git a/fetch/sum.go b/fetch/sum.go deleted file mode 100644 index 7386d1b..0000000 --- a/fetch/sum.go +++ /dev/null @@ -1,34 +0,0 @@ -package fetch - -import ( - "bytes" - "fmt" - "io/ioutil" - "strings" -) - -func parseGoSum(file string) (map[string]string, error) { - - // Read go.mod - data, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - pkgs := make(map[string]string) // goPackagepath -> rev - for lineno, line := range bytes.Split(data, []byte("\n")) { - if len(line) == 0 { - continue - } - - f := strings.Fields(string(line)) - if len(f) != 3 { - return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f)) - } - - pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod") - } - - return pkgs, nil - -} diff --git a/formats/gomod2nix/gomod2nix.go b/formats/gomod2nix/gomod2nix.go index 1e38a82..4577ac6 100644 --- a/formats/gomod2nix/gomod2nix.go +++ b/formats/gomod2nix/gomod2nix.go @@ -2,84 +2,43 @@ package gomod2nix import ( "bytes" - "fmt" "github.com/BurntSushi/toml" "github.com/tweag/gomod2nix/types" - "io/ioutil" ) -type fetchInfo struct { - Type string `toml:"type"` - URL string `toml:"url"` - Rev string `toml:"rev"` - Sha256 string `toml:"sha256"` -} +const schemaVersion = 1 type packageT struct { - SumVersion string `toml:"sumVersion"` - RelPath string `toml:"relPath,omitempty"` - VendorPath string `toml:"vendorPath,omitempty"` - Fetch *fetchInfo `toml:"fetch"` + Version string `toml:"version"` + Hash string `toml:"hash"` + ReplacedPath string `toml:"replaced,omitempty"` +} + +type output struct { + SchemaVersion int `toml:"schema"` + Mod map[string]*packageT `toml:"mod"` } func Marshal(pkgs []*types.Package) ([]byte, error) { - result := make(map[string]*packageT) + out := &output{ + SchemaVersion: schemaVersion, + Mod: make(map[string]*packageT), + } for _, pkg := range pkgs { - result[pkg.GoPackagePath] = &packageT{ - VendorPath: pkg.VendorPath, - SumVersion: pkg.SumVersion, - RelPath: pkg.RelPath, - Fetch: &fetchInfo{ - Type: "git", - URL: pkg.URL, - Rev: pkg.Rev, - Sha256: pkg.Sha256, - }, + out.Mod[pkg.GoPackagePath] = &packageT{ + Version: pkg.Version, + Hash: pkg.Hash, + ReplacedPath: pkg.ReplacedPath, } } var buf bytes.Buffer e := toml.NewEncoder(&buf) - err := e.Encode(result) + err := e.Encode(out) if err != nil { return nil, err } return buf.Bytes(), nil } - -func LoadGomod2Nix(filePath string) map[string]*types.Package { - ret := make(map[string]*types.Package) - - if filePath == "" { - return ret - } - - b, err := ioutil.ReadFile(filePath) - if err != nil { - fmt.Println(err) - return ret - } - - result := make(map[string]*packageT) - _, err = toml.Decode(string(b), &result) - if err != nil { - fmt.Println(err) - return ret - } - - for k, v := range result { - ret[k] = &types.Package{ - GoPackagePath: k, - URL: v.Fetch.URL, - Rev: v.Fetch.Rev, - Sha256: v.Fetch.Sha256, - SumVersion: v.SumVersion, - RelPath: v.RelPath, - VendorPath: v.VendorPath, - } - } - - return ret -} diff --git a/types/types.go b/types/types.go index 6aad1fc..9d9ce0e 100644 --- a/types/types.go +++ b/types/types.go @@ -2,10 +2,7 @@ package types type Package struct { GoPackagePath string - URL string - Rev string - Sha256 string - SumVersion string - RelPath string - VendorPath string + ReplacedPath string + Version string + Hash string }