fetch: Migrate to new fetcher method (using Go as the fetcher)

fetch/fetch.go

@@ -1,22 +1,18 @@
 package fetch
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
-	"os"
 	"os/exec"
-	"path/filepath"
+	"path"
-	"regexp"
-	"sort"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/tweag/gomod2nix/formats/gomod2nix"
 	"github.com/tweag/gomod2nix/types"
 	"golang.org/x/mod/modfile"
-	"golang.org/x/mod/module"
-	"golang.org/x/tools/go/vcs"
 )
 
 type packageJob struct {
@@ -30,21 +26,15 @@ type packageResult struct {
 	err error
 }
 
-func worker(id int, cache map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) {
+type goModDownload struct {
-	log.WithField("workerId", id).Info("Starting worker process")
+	Path     string
-
+	Version  string
-	for j := range jobs {
+	Info     string
-		log.WithFields(log.Fields{
+	GoMod    string
-			"workerId":      id,
+	Zip      string
-			"goPackagePath": j.goPackagePath,
+	Dir      string
-		}).Info("Worker received job")
+	Sum      string
-
+	GoModSum string
-		pkg, err := fetchPackage(cache, j.importPath, j.goPackagePath, j.sumVersion)
-		results <- &packageResult{
-			err: err,
-			pkg: pkg,
-		}
-	}
 }
 
 func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
@@ -65,198 +55,80 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, num
 		return nil, err
 	}
 
-	caches := gomod2nix.LoadGomod2Nix(goMod2NixPath)
-
 	// Map repos -> replacement repo
 	replace := make(map[string]string)
 	for _, repl := range mod.Replace {
 		replace[repl.New.Path] = repl.Old.Path
 	}
 
+	var modDownloads []*goModDownload
+	{
 		log.WithFields(log.Fields{
 			"sumPath": goSumPath,
-	}).Info("Parsing go.sum")
+		}).Info("Downloading dependencies")
 
-	sumVersions, err := parseGoSum(goSumPath)
-	if err != nil {
-		return nil, err
-	}
-
-	numJobs := len(sumVersions)
-	if numJobs < numWorkers {
-		numWorkers = numJobs
-	}
-
-	log.WithFields(log.Fields{
-		"numWorkers": numWorkers,
-	}).Info("Starting worker processes")
-	jobs := make(chan *packageJob, numJobs)
-	results := make(chan *packageResult, numJobs)
-	for i := 0; i <= numWorkers; i++ {
-		go worker(i, caches, jobs, results)
-	}
-
-	log.WithFields(log.Fields{
-		"numJobs": numJobs,
-	}).Info("Queuing jobs")
-	for importPath, sumVersion := range sumVersions {
-		// Check for replacement path (only original goPackagePath is recorded in go.sum)
-		goPackagePath := importPath
-		v, ok := replace[goPackagePath]
-		if ok {
-			goPackagePath = v
-		}
-
-		jobs <- &packageJob{
-			importPath:    importPath,
-			goPackagePath: goPackagePath,
-			sumVersion:    sumVersion,
-		}
-	}
-	close(jobs)
-
-	var pkgs []*types.Package
-	for i := 1; i <= numJobs; i++ {
-		result := <-results
-
-		log.WithFields(log.Fields{
-			"current": i,
-			"total":   numJobs,
-		}).Info("Received finished job")
-
-		if result.err != nil {
-			if keepGoing {
-				fmt.Println(result.err)
-				continue
-			} else {
-				return nil, result.err
-			}
-		}
-
-		pkgs = append(pkgs, result.pkg)
-	}
-
-	sort.Slice(pkgs, func(i, j int) bool {
-		return pkgs[i].GoPackagePath < pkgs[j].GoPackagePath
-	})
-
-	return pkgs, nil
-}
-
-func fetchPackage(cache map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) {
-	repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
-	if err != nil {
-		return nil, err
-	}
-
-	commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z\-]*?[0-9]{14}-(.*?)$`)
-	rev := strings.TrimSuffix(sumVersion, "+incompatible")
-	if commitShaRev.MatchString(rev) {
-		rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
-	}
-
-	importPathPrefix, pathMajor, _ := module.SplitPathVersion(importPath)
-
-	// Relative path within the repo
-	relPath := strings.TrimPrefix(importPathPrefix, repoRoot.Root+"/")
-	if relPath == importPathPrefix {
-		relPath = ""
-	}
-
-	log.WithFields(log.Fields{
-		"goPackagePath": goPackagePath,
-	}).Info("Checking previous invocation cache")
-	{
-		cached, ok := cache[goPackagePath]
-		if ok {
-			if cached.SumVersion == sumVersion {
-				log.WithFields(log.Fields{
-					"goPackagePath": goPackagePath,
-				}).Info("Returning cached entry")
-				return cached, nil
-			}
-		}
-	}
-
-	if repoRoot.VCS.Name != "Git" {
-		return nil, fmt.Errorf("Only git repositories are supported")
-	}
-
-	type prefetchOutput struct {
-		URL    string `json:"url"`
-		Rev    string `json:"rev"`
-		Sha256 string `json:"sha256"`
-		Path   string `json:"path"`
-		// date   string
-		// fetchSubmodules bool
-		// deepClone       bool
-		// leaveDotGit     bool
-	}
-
-	log.WithFields(log.Fields{
-		"goPackagePath": goPackagePath,
-		"rev":           rev,
-	}).Info("Cache miss, fetching")
 		stdout, err := exec.Command(
-		"nix-prefetch-git",
+			"go", "mod", "download", "--json",
-		"--quiet",
+		).Output()
-		"--fetch-submodules",
-		"--url", repoRoot.Repo,
-		"--rev", rev).Output()
-	if err != nil {
-		newRev := fmt.Sprintf("%s/%s", relPath, rev)
-
-		log.WithFields(log.Fields{
-			"goPackagePath": goPackagePath,
-			"rev":           newRev,
-		}).Info("Fetching failed, retrying with different rev format")
-		originalErr := err
-		stdout, err = exec.Command(
-			"nix-prefetch-git",
-			"--quiet",
-			"--fetch-submodules",
-			"--url", repoRoot.Repo,
-			"--rev", newRev).Output()
-		if err != nil {
-			log.WithFields(log.Fields{
-				"goPackagePath": goPackagePath,
-			}).Error("Fetching failed")
-			return nil, originalErr
-		}
-
-		rev = newRev
-	}
-
-	var output *prefetchOutput
-	err = json.Unmarshal(stdout, &output)
 		if err != nil {
 			return nil, err
 		}
 
-	vendorPath := ""
+		dec := json.NewDecoder(bytes.NewReader(stdout))
-	if importPath != goPackagePath {
+		for {
-		vendorPath = importPath
+			var dl *goModDownload
+			err := dec.Decode(&dl)
+			if err == io.EOF {
+				break
+			}
+			modDownloads = append(modDownloads, dl)
 		}
 
-	if relPath == "" && pathMajor != "" {
+		log.WithFields(log.Fields{
-		p := filepath.Join(output.Path, pathMajor)
+			"sumPath": goSumPath,
-		_, err := os.Stat(p)
+		}).Info("Done downloading dependencies")
-		if err == nil {
-			fmt.Println(pathMajor)
-			relPath = strings.TrimPrefix(pathMajor, "/")
-		}
 	}
 
-	return &types.Package{
+	packages := []*types.Package{}
+	for _, dl := range modDownloads {
+
+		goPackagePath, hasReplace := replace[dl.Path]
+		if !hasReplace {
+			goPackagePath = dl.Path
+		}
+
+		var storePath string
+		{
+			stdout, err := exec.Command(
+				"nix", "eval", "--impure", "--expr",
+				fmt.Sprintf("builtins.path { name = \"%s_%s\"; path = \"%s\"; }", path.Base(goPackagePath), dl.Version, dl.Dir),
+			).Output()
+			if err != nil {
+				return nil, err
+			}
+			storePath = string(stdout)[1 : len(stdout)-2]
+		}
+
+		stdout, err := exec.Command(
+			"nix-store", "--query", "--hash", storePath,
+		).Output()
+		if err != nil {
+			return nil, err
+		}
+		hash := strings.TrimSpace(string(stdout))
+
+		pkg := &types.Package{
 			GoPackagePath: goPackagePath,
-		URL:           repoRoot.Repo,
+			Version:       dl.Version,
-		Rev:           output.Rev,
+			Hash:          hash,
-		Sha256:        output.Sha256,
+		}
-		// This is used to skip fetching where the previous package path & versions are still the same
+		if hasReplace {
-		// It's also used to construct the vendor directory in the Nix build
+			pkg.ReplacedPath = dl.Path
-		SumVersion: sumVersion,
+		}
-		RelPath:    relPath,
+
-		VendorPath: vendorPath,
+		packages = append(packages, pkg)
-	}, nil
+	}
+
+	return packages, nil
 
 }

fetch/sum.go

@@ -1,34 +0,0 @@
-package fetch
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"strings"
-)
-
-func parseGoSum(file string) (map[string]string, error) {
-
-	// Read go.mod
-	data, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
-
-	pkgs := make(map[string]string) // goPackagepath -> rev
-	for lineno, line := range bytes.Split(data, []byte("\n")) {
-		if len(line) == 0 {
-			continue
-		}
-
-		f := strings.Fields(string(line))
-		if len(f) != 3 {
-			return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
-		}
-
-		pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod")
-	}
-
-	return pkgs, nil
-
-}

formats/gomod2nix/gomod2nix.go

@@ -2,84 +2,43 @@ package gomod2nix
 
 import (
 	"bytes"
-	"fmt"
 	"github.com/BurntSushi/toml"
 	"github.com/tweag/gomod2nix/types"
-	"io/ioutil"
 )
 
-type fetchInfo struct {
+const schemaVersion = 1
-	Type   string `toml:"type"`
-	URL    string `toml:"url"`
-	Rev    string `toml:"rev"`
-	Sha256 string `toml:"sha256"`
-}
 
 type packageT struct {
-	SumVersion string     `toml:"sumVersion"`
+	Version      string `toml:"version"`
-	RelPath    string     `toml:"relPath,omitempty"`
+	Hash         string `toml:"hash"`
-	VendorPath string     `toml:"vendorPath,omitempty"`
+	ReplacedPath string `toml:"replaced,omitempty"`
-	Fetch      *fetchInfo `toml:"fetch"`
+}
+
+type output struct {
+	SchemaVersion int                  `toml:"schema"`
+	Mod           map[string]*packageT `toml:"mod"`
 }
 
 func Marshal(pkgs []*types.Package) ([]byte, error) {
-	result := make(map[string]*packageT)
+	out := &output{
+		SchemaVersion: schemaVersion,
+		Mod:           make(map[string]*packageT),
+	}
 
 	for _, pkg := range pkgs {
-		result[pkg.GoPackagePath] = &packageT{
+		out.Mod[pkg.GoPackagePath] = &packageT{
-			VendorPath: pkg.VendorPath,
+			Version:      pkg.Version,
-			SumVersion: pkg.SumVersion,
+			Hash:         pkg.Hash,
-			RelPath:    pkg.RelPath,
+			ReplacedPath: pkg.ReplacedPath,
-			Fetch: &fetchInfo{
-				Type:   "git",
-				URL:    pkg.URL,
-				Rev:    pkg.Rev,
-				Sha256: pkg.Sha256,
-			},
 		}
 	}
 
 	var buf bytes.Buffer
 	e := toml.NewEncoder(&buf)
-	err := e.Encode(result)
+	err := e.Encode(out)
 	if err != nil {
 		return nil, err
 	}
 
 	return buf.Bytes(), nil
 }
-
-func LoadGomod2Nix(filePath string) map[string]*types.Package {
-	ret := make(map[string]*types.Package)
-
-	if filePath == "" {
-		return ret
-	}
-
-	b, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		fmt.Println(err)
-		return ret
-	}
-
-	result := make(map[string]*packageT)
-	_, err = toml.Decode(string(b), &result)
-	if err != nil {
-		fmt.Println(err)
-		return ret
-	}
-
-	for k, v := range result {
-		ret[k] = &types.Package{
-			GoPackagePath: k,
-			URL:           v.Fetch.URL,
-			Rev:           v.Fetch.Rev,
-			Sha256:        v.Fetch.Sha256,
-			SumVersion:    v.SumVersion,
-			RelPath:       v.RelPath,
-			VendorPath:    v.VendorPath,
-		}
-	}
-
-	return ret
-}

types/types.go

@@ -2,10 +2,7 @@ package types
 
 type Package struct {
 	GoPackagePath string
-	URL           string
+	ReplacedPath  string
-	Rev           string
+	Version       string
-	Sha256        string
+	Hash          string
-	SumVersion    string
-	RelPath       string
-	VendorPath    string
 }