Basic parsing/fetching working

fetch.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"golang.org/x/tools/go/vcs"
+	"os/exec"
+)
+
+type Package struct {
+	GoPackagePath string
+	URL           string
+	Rev           string
+	Hash          string
+}
+
+func fetchPackage(replace map[string]string, goPackagePath string, rev string) (*Package, error) {
+
+	// Check for replacement path (only original goPackagePath is recorded in go.sum)
+	repo := goPackagePath
+	v, ok := replace[goPackagePath]
+	if ok {
+		repo = v
+	}
+
+	repoRoot, err := vcs.RepoRootForImportPath(repo, false)
+	if err != nil {
+		return nil, err
+	}
+
+	if repoRoot.VCS.Name != "Git" {
+		return nil, fmt.Errorf("Only git repositories are supported")
+	}
+
+	type prefetchOutput struct {
+		URL    string `json:"url"`
+		Rev    string `json:"rev"`
+		Sha256 string `json:"sha256"`
+		// path   string
+		// date   string
+		// fetchSubmodules bool
+		// deepClone       bool
+		// leaveDotGit     bool
+	}
+	stdout, err := exec.Command(
+		"nix-prefetch-git",
+		"--quiet",
+		"--fetch-submodules",
+		"--url", repoRoot.Repo,
+		"--rev", rev).Output()
+	if err != nil {
+		return nil, err
+	}
+
+	var output *prefetchOutput
+
+	err = json.Unmarshal(stdout, &output)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Package{
+		GoPackagePath: goPackagePath,
+		URL:           repoRoot.Repo,
+		// It may feel appealing to use output.Rev to get the full git hash
+		// However, this has the major downside of not being able to be checked against an
+		// older output file (as the revs) don't match
+		//
+		// This is used to skip fetching where the previous package path & rev are still the same
+		Rev:  rev,
+		Hash: fmt.Sprintf("sha256:%s", output.Sha256),
+	}, nil
+
+}

go.mod

@@ -0,0 +1,8 @@
+module github.com/tweag/gomod2nix
+
+go 1.14
+
+require (
+	golang.org/x/mod v0.3.0
+	golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
+)

go.sum

@@ -0,0 +1,15 @@
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

@@ -0,0 +1,94 @@
+package main // import "github.com/tweag/gomod2nix"
+
+import (
+	"flag"
+	"fmt"
+	"golang.org/x/mod/modfile"
+	"io/ioutil"
+	"path/filepath"
+)
+
+type packageJob struct {
+	goPackagePath string
+	rev           string
+}
+
+type packageResult struct {
+	pkg *Package
+	err error
+}
+
+func worker(id int, replace map[string]string, jobs <-chan *packageJob, results chan<- *packageResult) {
+	for j := range jobs {
+		pkg, err := fetchPackage(replace, j.goPackagePath, j.rev)
+		results <- &packageResult{
+			err: err,
+			pkg: pkg,
+		}
+	}
+}
+
+func main() {
+
+	// var jobs = flag.Int("jobs", 20, "Number of parallel jobs")
+	flag.Parse()
+
+	numWorkers := 10
+
+	// directory := "testdata"
+	directory := "./"
+
+	// Read go.mod
+	data, err := ioutil.ReadFile(filepath.Join(directory, "go.mod"))
+	if err != nil {
+		panic(err)
+	}
+
+	// Parse go.mod
+	mod, err := modfile.Parse("go.mod", data, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// // Parse require
+	// require := make(map[string]module.Version)
+	// for _, req := range mod.Require {
+	// 	require[req.Mod.Path] = req.Mod
+	// }
+
+	// Map repos -> replacement repo
+	replace := make(map[string]string)
+	for _, repl := range mod.Replace {
+		replace[repl.Old.Path] = repl.New.Path
+	}
+
+	revs, err := parseGoSum(filepath.Join(directory, "go.sum"))
+	if err != nil {
+		panic(err)
+	}
+
+	numJobs := len(revs)
+	if numJobs < numWorkers {
+		numWorkers = numJobs
+	}
+
+	jobs := make(chan *packageJob, numJobs)
+	results := make(chan *packageResult, numJobs)
+	for i := 0; i <= numWorkers; i++ {
+		go worker(i, replace, jobs, results)
+	}
+
+	for goPackagePath, rev := range revs {
+		jobs <- &packageJob{
+			goPackagePath: goPackagePath,
+			rev:           rev,
+		}
+	}
+	close(jobs)
+
+	for i := 1; i <= numJobs; i++ {
+		result := <-results
+		fmt.Println(result)
+	}
+
+}

sum.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strings"
+)
+
+func parseGoSum(file string) (map[string]string, error) {
+	commitShaRev := regexp.MustCompile(`^v\d+\.\d+\.\d+-(?:\d+\.)?[0-9]{14}-(.*?)$`)
+
+	// Read go.mod
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	pkgs := make(map[string]string) // goPackagepath -> rev
+	for lineno, line := range bytes.Split(data, []byte("\n")) {
+		if len(line) == 0 {
+			continue
+		}
+
+		f := strings.Fields(string(line))
+		if len(f) != 3 {
+			return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
+		}
+
+		rev := strings.TrimSuffix(strings.TrimSuffix(f[1], "/go.mod"), "+incompatible")
+		if commitShaRev.MatchString(rev) {
+			rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
+		}
+
+		pkgs[f[0]] = rev
+	}
+
+	return pkgs, nil
+
+}