Start changing the format to work in the Nix sandbox

fetch/fetch.go

@@ -8,9 +8,11 @@ import (
 	"github.com/tweag/gomod2nix/formats/gomod2nix"
 	"github.com/tweag/gomod2nix/types"
 	"golang.org/x/mod/modfile"
+	"golang.org/x/mod/module"
 	"golang.org/x/tools/go/vcs"
 	"io/ioutil"
 	"os/exec"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -18,7 +20,7 @@ import (
 type packageJob struct {
 	importPath    string
 	goPackagePath string
-	rev           string
+	sumVersion    string
 }
 
 type packageResult struct {
@@ -35,7 +37,7 @@ func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob,
 			"goPackagePath": j.goPackagePath,
 		}).Info("Worker received job")
 
-		pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.rev)
+		pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.sumVersion)
 		results <- &packageResult{
 			err: err,
 			pkg: pkg,
@@ -43,11 +45,6 @@ func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob,
 	}
 }
 
-// It's a relatively common idiom to tag storage/v1.0.0
-func mkNewRev(goPackagePath string, repoRoot *vcs.RepoRoot, rev string) string {
-	return fmt.Sprintf("%s/%s", strings.TrimPrefix(goPackagePath, repoRoot.Root+"/"), rev)
-}
-
 func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, depsNixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
 
 	log.WithFields(log.Fields{
@@ -92,12 +89,12 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
 		"sumPath": goSumPath,
 	}).Info("Parsing go.sum")
 
-	revs, err := parseGoSum(goSumPath)
+	sumVersions, err := parseGoSum(goSumPath)
 	if err != nil {
 		return nil, err
 	}
 
-	numJobs := len(revs)
+	numJobs := len(sumVersions)
 	if numJobs < numWorkers {
 		numWorkers = numJobs
 	}
@@ -114,17 +111,18 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
 	log.WithFields(log.Fields{
 		"numJobs": numJobs,
 	}).Info("Queuing jobs")
-	for goPackagePath, rev := range revs {
+	for goPackagePath, sumVersion := range sumVersions {
 		// Check for replacement path (only original goPackagePath is recorded in go.sum)
 		importPath := goPackagePath
 		v, ok := replace[goPackagePath]
 		if ok {
 			importPath = v
 		}
+
 		jobs <- &packageJob{
 			importPath:    importPath,
 			goPackagePath: goPackagePath,
-			rev:           rev,
+			sumVersion:    sumVersion,
 		}
 	}
 	close(jobs)
@@ -157,13 +155,27 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
 	return pkgs, nil
 }
 
-func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, rev string) (*types.Package, error) {
+func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) {
 	repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
 	if err != nil {
 		return nil, err
 	}
 
-	newRev := mkNewRev(goPackagePath, repoRoot, rev)
+	commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z]*?[0-9]{14}-(.*?)$`)
+	rev := strings.TrimSuffix(sumVersion, "+incompatible")
+	if commitShaRev.MatchString(rev) {
+		rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
+	}
+
+	goPackagePathPrefix, _, _ := module.SplitPathVersion(goPackagePath)
+
+	// Relative path within the repo
+	relPath := strings.TrimPrefix(goPackagePathPrefix, repoRoot.Root+"/")
+	if relPath == goPackagePathPrefix {
+		relPath = ""
+	}
+
+	newRev := fmt.Sprintf("%s/%s", relPath, rev)
 	if len(caches) > 0 {
 
 		log.WithFields(log.Fields{
@@ -242,13 +254,12 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
 	return &types.Package{
 		GoPackagePath: goPackagePath,
 		URL:           repoRoot.Repo,
-		// It may feel appealing to use output.Rev to get the full git hash
+		Rev:           output.Rev,
-		// However, this has the major downside of not being able to be checked against an
+		Sha256:        output.Sha256,
-		// older output file (as the revs) don't match
+		// This is used to skip fetching where the previous package path & versions are still the same
-		//
+		// It's also used to construct the vendor directory in the Nix build
-		// This is used to skip fetching where the previous package path & rev are still the same
+		SumVersion: sumVersion,
-		Rev:    rev,
+		RelPath:    relPath,
-		Sha256: output.Sha256,
 	}, nil
 
 }

fetch/sum.go

@@ -4,12 +4,10 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
-	"regexp"
 	"strings"
 )
 
 func parseGoSum(file string) (map[string]string, error) {
-	commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z]*?[0-9]{14}-(.*?)$`)
 
 	// Read go.mod
 	data, err := ioutil.ReadFile(file)
@@ -28,12 +26,7 @@ func parseGoSum(file string) (map[string]string, error) {
 			return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
 		}
 
-		rev := strings.TrimSuffix(strings.TrimSuffix(f[1], "/go.mod"), "+incompatible")
+		pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod")
-		if commitShaRev.MatchString(rev) {
-			rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
-		}
-
-		pkgs[f[0]] = rev
 	}
 
 	return pkgs, nil

formats/gomod2nix/gomod2nix.go

@@ -8,22 +8,32 @@ import (
 	"io/ioutil"
 )
 
-type packageT struct {
+type fetchInfo struct {
 	Type   string `toml:"type"`
 	URL    string `toml:"url"`
 	Rev    string `toml:"rev"`
 	Sha256 string `toml:"sha256"`
 }
 
+type packageT struct {
+	SumVersion string     `toml:"sumVersion"`
+	RelPath    string     `toml:"relPath"`
+	Fetch      *fetchInfo `toml:"fetch"`
+}
+
 func Marshal(pkgs []*types.Package) ([]byte, error) {
 	result := make(map[string]*packageT)
 
 	for _, pkg := range pkgs {
 		result[pkg.GoPackagePath] = &packageT{
-			Type:   "git",
+			SumVersion: pkg.SumVersion,
-			URL:    pkg.URL,
+			RelPath:    pkg.RelPath,
-			Rev:    pkg.Rev,
+			Fetch: &fetchInfo{
-			Sha256: pkg.Sha256,
+				Type:   "git",
+				URL:    pkg.URL,
+				Rev:    pkg.Rev,
+				Sha256: pkg.Sha256,
+			},
 		}
 	}
 
@@ -60,9 +70,11 @@ func LoadGomod2Nix(filePath string) map[string]*types.Package {
 	for k, v := range result {
 		ret[k] = &types.Package{
 			GoPackagePath: k,
-			URL:           v.URL,
+			URL:           v.Fetch.URL,
-			Rev:           v.Rev,
+			Rev:           v.Fetch.Rev,
-			Sha256:        v.Sha256,
+			Sha256:        v.Fetch.Sha256,
+			SumVersion:    v.SumVersion,
+			RelPath:       v.RelPath,
 		}
 	}
 

types/types.go

@@ -5,4 +5,6 @@ type Package struct {
 	URL           string
 	Rev           string
 	Sha256        string
+	SumVersion    string
+	RelPath       string
 }