Merge pull request #41 from adisbladis/go-mod-download-fetcher

Use `go mod download` as a fetcher

.github/workflows/ci.yml

@@ -17,18 +17,48 @@ jobs:
     - uses: cachix/install-nix-action@v12
     - uses: actions/checkout@v1
     - name: Check format
-      run: ./check-fmt
+      run: nix-shell --run 'nixpkgs-fmt --check .'
 
-  build:
+  gomod2nix_toml:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        nixpkgs_version: ["master"]
-        # os: [ubuntu-latest, macos-latest]
     env:
       NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz"
     steps:
     - uses: cachix/install-nix-action@v12
     - uses: actions/checkout@v1
-    - run: nix-shell --run 'make'
+    - name: "Build gomod2nix"
+      run: nix-shell --run "go build"
+    - name: Run gomod2nix
+      run: nix-shell --run gomod2nix
+    - name: Check diff
+      run: git diff --exit-code gomod2nix.toml
+
+  list-jobs:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v17
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - id: set-matrix
+        run: |
+          set -euo pipefail
+          matrix="$(go run tests/run.go list | jq --raw-input --slurp -rcM '{attr: split("\n")[:-1], os: ["ubuntu-latest"]}')"
+          echo "::set-output name=matrix::$matrix"
+
+  builds:
+    needs: list-jobs
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix: ${{fromJSON(needs.list-jobs.outputs.matrix)}}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: cachix/install-nix-action@v17
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: "Build gomod2nix"
+        run: nix-shell --run "go build"
+      - name: "Run test: ${{ matrix.attr }}"
+        run: nix-shell --run "go run tests/run.go run ${{ matrix.attr }}"

.github/workflows/test.yml

@@ -1,20 +0,0 @@
-name: Test
-
-on:
-  pull_request:
-    paths-ignore:
-    - '**.md'
-  push:
-    paths-ignore:
-    - '**.md'
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    env:
-      NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-unstable.tar.gz"
-    steps:
-    - uses: cachix/install-nix-action@v12
-    - uses: actions/checkout@v1
-    - run: |
-      make -C tests

Makefile

@@ -1,3 +0,0 @@
-all:
-	go build
-	$(MAKE) -C tests

builder/default.nix

@@ -1,10 +1,13 @@
 { stdenv
+, stdenvNoCC
 , runCommand
 , buildEnv
 , lib
 , fetchgit
 , removeReferencesTo
-, go
+, jq
+, cacert
+, pkgs
 }:
 let
 
@@ -12,8 +15,27 @@ let
 
   removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
 
+  fetchGoModule =
+    { hash
+    , goPackagePath
+    , version
+    , go ? pkgs.go
+    }:
+    stdenvNoCC.mkDerivation {
+      name = "${baseNameOf goPackagePath}_${version}";
+      builder = ./fetch.sh;
+      inherit goPackagePath version;
+      nativeBuildInputs = [ go jq ];
+      outputHashMode = "recursive";
+      outputHashAlgo = null;
+      outputHash = hash;
+      SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+      impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+    };
+
   buildGoApplication =
     { modules
+    , go ? pkgs.go
     , src
     , pwd ? null
     , CGO_ENABLED ? "0"
@@ -47,17 +69,15 @@ let
           nativeBuildInputs = [ go ];
           json = builtins.toJSON modulesStruct;
 
-          sources = builtins.toJSON (lib.mapAttrs
+          sources = builtins.toJSON (
-            (goPackagePath: meta:
+            lib.mapAttrs
-              let
+              (goPackagePath: meta: fetchGoModule {
-                src = fetchgit {
+                goPackagePath = meta.replaced or goPackagePath;
-                  inherit (meta.fetch) url sha256 rev;
+                inherit (meta) version hash;
-                  fetchSubmodules = true;
+                inherit go;
-                };
+              })
-                srcPath = "${src}/${meta.relPath or ""}";
+              modulesStruct.mod
-              in
+          );
-              srcPath)
-            modulesStruct);
 
           passAsFile = [ "json" "sources" ];
         }

builder/fetch.sh

@@ -0,0 +1,10 @@
+source $stdenv/setup
+
+export HOME=$(mktemp -d)
+
+# Call once first outside of subshell for better error reporting
+go mod download "$goPackagePath@$version"
+
+dir=$(go mod download --json "$goPackagePath@$version" | jq -r .Dir)
+
+cp -r $dir $out

builder/parser.nix

@@ -3,7 +3,7 @@
 # in normalised form.
 
 let
-  inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf;
+  inherit (builtins) elemAt mapAttrs split foldl' match filter typeOf hasAttr length;
 
   # Strip lines with comments & other junk
   stripStr = s: elemAt (split "^ *" (elemAt (split " *$" s) 0)) 2;
@@ -16,9 +16,6 @@ let
     # Strip leading tabs characters
     (lines: map (l: elemAt (match "(\t)?(.*)" l) 1) lines)
 
-    # Strip comment lines
-    (filter (l: match "[ \t]*//.*" l != null))
-
     # Filter empty lines
     (filter (l: l != ""))
   ];
@@ -40,21 +37,40 @@ let
 
       in
       {
-        data = acc.data // (
+        data = (acc.data // (
           if directive == "" && rest == ")" then { }
-          else if inDirective != null && rest == "(" then {
+          else if inDirective != null && rest == "(" && ! hasAttr inDirective acc.data then {
             ${inDirective} = { };
-          } else if inDirective != null then {
+          }
+          else if rest == "(" || rest == ")" then { }
+          else if inDirective != null then {
             ${inDirective} = acc.data.${inDirective} // { ${directive} = rest; };
-          } else {
+          } else if directive == "replace" then
+            (
+              let
+                segments = split " => " rest;
+                getSegment = elemAt segments;
+              in
+              assert length segments == 3; {
+                replace = acc.data.replace // {
+                  ${getSegment 0} = "=> ${getSegment 2}";
+                };
+              }
+            )
+          else {
             ${directive} = rest;
           }
+        )
         );
         inherit inDirective;
       })
     {
       inDirective = null;
-      data = { };
+      data = {
+        require = { };
+        replace = { };
+        exclude = { };
+      };
     }
     lines
   ).data;
@@ -98,14 +114,14 @@ let
     data // {
       replace =
         mapAttrs
-          (n: v:
+          (_: v:
             let
               m = match "=> ([^ ]+) (.+)" v;
               m2 = match "=> (.*+)" v;
             in
             if m != null then {
               goPackagePath = elemAt m 0;
-              version = parseVersion (elemAt m 1);
+              version = elemAt m 1;
             } else {
               path = elemAt m2 0;
             })
@@ -113,12 +129,6 @@ let
     }
   );
 
-  parseRequire = data: (
-    data // {
-      require = mapAttrs (n: v: parseVersion v) data.require;
-    }
-  );
-
   splitString = sep: s: filter (t: t != [ ]) (split sep s);
 
 in
@@ -128,5 +138,4 @@ foldl' (acc: f: f acc) (splitString "\n" contents) [
   parseLines
   normaliseDirectives
   parseReplace
-  parseRequire
 ]

builder/symlink.go

@@ -9,23 +9,21 @@ import (
 	"sort"
 )
 
-type fetchInfo struct {
+type Package struct {
-	Type   string `toml:"type"`
+	GoPackagePath string `json:"-"`
-	URL    string `toml:"url"`
+	Version       string `json:"version"`
-	Rev    string `toml:"rev"`
+	Hash          string `json:"hash"`
-	Sha256 string `toml:"sha256"`
+	ReplacedPath  string `json:"replaced,omitempty"`
 }
 
-type packageT struct {
+type Output struct {
-	SumVersion string     `toml:"sumVersion"`
+	SchemaVersion int                 `json:"schema"`
-	RelPath    string     `toml:"relPath,omitempty"`
+	Mod           map[string]*Package `json:"mod"`
-	VendorPath string     `toml:"vendorPath,omitempty"`
-	Fetch      *fetchInfo `toml:"fetch"`
 }
 
 func main() {
 
-	pkgs := make(map[string]*packageT)
+	var output Output
 	sources := make(map[string]string)
 
 	b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
@@ -43,11 +41,13 @@ func main() {
 		panic(err)
 	}
 
-	err = json.Unmarshal(b, &pkgs)
+	err = json.Unmarshal(b, &output)
 	if err != nil {
 		panic(err)
 	}
 
+	pkgs := output.Mod
+
 	keys := make([]string, 0, len(pkgs))
 	for key := range pkgs {
 		keys = append(keys, key)
@@ -58,12 +58,8 @@ func main() {
 	for i := len(keys) - 1; i >= 0; i-- {
 		key := keys[i]
 		src := sources[key]
-		pkg := pkgs[key]
 
 		paths := []string{key}
-		if pkg.VendorPath != "" {
-			paths = append(paths, pkg.VendorPath)
-		}
 
 		for _, path := range paths {
 

check-fmt

@@ -1,10 +0,0 @@
-#!/usr/bin/env nix-shell
-#!nix-shell ./shell.nix -i bash
-#
-# Just because the nixpkgs-fmt error message is not super readable. Used by
-# CI.
-
-if ! nixpkgs-fmt --check . ; then
-  echo 'run `nixpkgs-fmt .` to fix it'
-  exit 1
-fi

default.nix

@@ -1,6 +1,10 @@
-{ buildGoApplication, lib, makeWrapper, nix-prefetch-git }:
+{ buildGoApplication, go_1_18, nix, lib, makeWrapper, nix-prefetch-git }:
 
+let
+  go = go_1_18;
+in
 buildGoApplication {
+  inherit go;
   pname = "gomod2nix";
   version = "0.1";
   src = lib.cleanSourceWith {
@@ -9,12 +13,14 @@ buildGoApplication {
   };
   modules = ./gomod2nix.toml;
 
+  allowGoReference = true;
+
   subPackages = [ "." ];
 
   nativeBuildInputs = [ makeWrapper ];
 
   postInstall = ''
-    wrapProgram $out/bin/gomod2nix --prefix PATH : ${lib.makeBinPath [ nix-prefetch-git ]}
+    wrapProgram $out/bin/gomod2nix --prefix PATH : ${lib.makeBinPath [ go ]}
     rm -f $out/bin/builder
   '';
 }

fetch/fetch.go

@@ -1,269 +0,0 @@
-package fetch
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"regexp"
-	"sort"
-	"strings"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/tweag/gomod2nix/formats/gomod2nix"
-	"github.com/tweag/gomod2nix/types"
-	"golang.org/x/mod/modfile"
-	"golang.org/x/mod/module"
-	"golang.org/x/tools/go/vcs"
-)
-
-type packageJob struct {
-	importPath    string
-	goPackagePath string
-	sumVersion    string
-}
-
-type packageResult struct {
-	pkg *types.Package
-	err error
-}
-
-func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) {
-	log.WithField("workerId", id).Info("Starting worker process")
-
-	for j := range jobs {
-		log.WithFields(log.Fields{
-			"workerId":      id,
-			"goPackagePath": j.goPackagePath,
-		}).Info("Worker received job")
-
-		pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.sumVersion)
-		results <- &packageResult{
-			err: err,
-			pkg: pkg,
-		}
-	}
-}
-
-func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
-
-	log.WithFields(log.Fields{
-		"modPath": goModPath,
-	}).Info("Parsing go.mod")
-
-	// Read go.mod
-	data, err := ioutil.ReadFile(goModPath)
-	if err != nil {
-		return nil, err
-	}
-
-	// Parse go.mod
-	mod, err := modfile.Parse(goModPath, data, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	caches := []map[string]*types.Package{}
-	goModCache := gomod2nix.LoadGomod2Nix(goMod2NixPath)
-	if len(goModCache) > 0 {
-		caches = append(caches, goModCache)
-	}
-
-	// Map repos -> replacement repo
-	replace := make(map[string]string)
-	for _, repl := range mod.Replace {
-		replace[repl.New.Path] = repl.Old.Path
-	}
-
-	log.WithFields(log.Fields{
-		"sumPath": goSumPath,
-	}).Info("Parsing go.sum")
-
-	sumVersions, err := parseGoSum(goSumPath)
-	if err != nil {
-		return nil, err
-	}
-
-	numJobs := len(sumVersions)
-	if numJobs < numWorkers {
-		numWorkers = numJobs
-	}
-
-	log.WithFields(log.Fields{
-		"numWorkers": numWorkers,
-	}).Info("Starting worker processes")
-	jobs := make(chan *packageJob, numJobs)
-	results := make(chan *packageResult, numJobs)
-	for i := 0; i <= numWorkers; i++ {
-		go worker(i, caches, jobs, results)
-	}
-
-	log.WithFields(log.Fields{
-		"numJobs": numJobs,
-	}).Info("Queuing jobs")
-	for importPath, sumVersion := range sumVersions {
-		// Check for replacement path (only original goPackagePath is recorded in go.sum)
-		goPackagePath := importPath
-		v, ok := replace[goPackagePath]
-		if ok {
-			goPackagePath = v
-		}
-
-		jobs <- &packageJob{
-			importPath:    importPath,
-			goPackagePath: goPackagePath,
-			sumVersion:    sumVersion,
-		}
-	}
-	close(jobs)
-
-	var pkgs []*types.Package
-	for i := 1; i <= numJobs; i++ {
-		result := <-results
-
-		log.WithFields(log.Fields{
-			"current": i,
-			"total":   numJobs,
-		}).Info("Received finished job")
-
-		if result.err != nil {
-			if keepGoing {
-				fmt.Println(result.err)
-				continue
-			} else {
-				return nil, result.err
-			}
-		}
-
-		pkgs = append(pkgs, result.pkg)
-	}
-
-	sort.Slice(pkgs, func(i, j int) bool {
-		return pkgs[i].GoPackagePath < pkgs[j].GoPackagePath
-	})
-
-	return pkgs, nil
-}
-
-func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) {
-	repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
-	if err != nil {
-		return nil, err
-	}
-
-	commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z\-]*?[0-9]{14}-(.*?)$`)
-	rev := strings.TrimSuffix(sumVersion, "+incompatible")
-	if commitShaRev.MatchString(rev) {
-		rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
-	}
-
-	importPathPrefix, pathMajor, _ := module.SplitPathVersion(importPath)
-
-	// Relative path within the repo
-	relPath := strings.TrimPrefix(importPathPrefix, repoRoot.Root+"/")
-	if relPath == importPathPrefix {
-		relPath = ""
-	}
-
-	if len(caches) > 0 {
-		log.WithFields(log.Fields{
-			"goPackagePath": goPackagePath,
-		}).Info("Checking previous invocation cache")
-
-		for _, cache := range caches {
-			cached, ok := cache[goPackagePath]
-			if ok {
-				if cached.SumVersion == sumVersion {
-					log.WithFields(log.Fields{
-						"goPackagePath": goPackagePath,
-					}).Info("Returning cached entry")
-					return cached, nil
-				}
-			}
-		}
-	}
-
-	if repoRoot.VCS.Name != "Git" {
-		return nil, fmt.Errorf("Only git repositories are supported")
-	}
-
-	type prefetchOutput struct {
-		URL    string `json:"url"`
-		Rev    string `json:"rev"`
-		Sha256 string `json:"sha256"`
-		Path   string `json:"path"`
-		// date   string
-		// fetchSubmodules bool
-		// deepClone       bool
-		// leaveDotGit     bool
-	}
-
-	log.WithFields(log.Fields{
-		"goPackagePath": goPackagePath,
-		"rev":           rev,
-	}).Info("Cache miss, fetching")
-	stdout, err := exec.Command(
-		"nix-prefetch-git",
-		"--quiet",
-		"--fetch-submodules",
-		"--url", repoRoot.Repo,
-		"--rev", rev).Output()
-	if err != nil {
-		newRev := fmt.Sprintf("%s/%s", relPath, rev)
-
-		log.WithFields(log.Fields{
-			"goPackagePath": goPackagePath,
-			"rev":           newRev,
-		}).Info("Fetching failed, retrying with different rev format")
-		originalErr := err
-		stdout, err = exec.Command(
-			"nix-prefetch-git",
-			"--quiet",
-			"--fetch-submodules",
-			"--url", repoRoot.Repo,
-			"--rev", newRev).Output()
-		if err != nil {
-			log.WithFields(log.Fields{
-				"goPackagePath": goPackagePath,
-			}).Error("Fetching failed")
-			return nil, originalErr
-		}
-
-		rev = newRev
-	}
-
-	var output *prefetchOutput
-	err = json.Unmarshal(stdout, &output)
-	if err != nil {
-		return nil, err
-	}
-
-	vendorPath := ""
-	if importPath != goPackagePath {
-		vendorPath = importPath
-	}
-
-	if relPath == "" && pathMajor != "" {
-		p := filepath.Join(output.Path, pathMajor)
-		_, err := os.Stat(p)
-		if err == nil {
-			fmt.Println(pathMajor)
-			relPath = strings.TrimPrefix(pathMajor, "/")
-		}
-	}
-
-	return &types.Package{
-		GoPackagePath: goPackagePath,
-		URL:           repoRoot.Repo,
-		Rev:           output.Rev,
-		Sha256:        output.Sha256,
-		// This is used to skip fetching where the previous package path & versions are still the same
-		// It's also used to construct the vendor directory in the Nix build
-		SumVersion: sumVersion,
-		RelPath:    relPath,
-		VendorPath: vendorPath,
-	}, nil
-
-}

fetch/sum.go

@@ -1,34 +0,0 @@
-package fetch
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"strings"
-)
-
-func parseGoSum(file string) (map[string]string, error) {
-
-	// Read go.mod
-	data, err := ioutil.ReadFile(file)
-	if err != nil {
-		return nil, err
-	}
-
-	pkgs := make(map[string]string) // goPackagepath -> rev
-	for lineno, line := range bytes.Split(data, []byte("\n")) {
-		if len(line) == 0 {
-			continue
-		}
-
-		f := strings.Fields(string(line))
-		if len(f) != 3 {
-			return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
-		}
-
-		pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod")
-	}
-
-	return pkgs, nil
-
-}

formats/gomod2nix/gomod2nix.go

@@ -1,85 +0,0 @@
-package gomod2nix
-
-import (
-	"bytes"
-	"fmt"
-	"github.com/BurntSushi/toml"
-	"github.com/tweag/gomod2nix/types"
-	"io/ioutil"
-)
-
-type fetchInfo struct {
-	Type   string `toml:"type"`
-	URL    string `toml:"url"`
-	Rev    string `toml:"rev"`
-	Sha256 string `toml:"sha256"`
-}
-
-type packageT struct {
-	SumVersion string     `toml:"sumVersion"`
-	RelPath    string     `toml:"relPath,omitempty"`
-	VendorPath string     `toml:"vendorPath,omitempty"`
-	Fetch      *fetchInfo `toml:"fetch"`
-}
-
-func Marshal(pkgs []*types.Package) ([]byte, error) {
-	result := make(map[string]*packageT)
-
-	for _, pkg := range pkgs {
-		result[pkg.GoPackagePath] = &packageT{
-			VendorPath: pkg.VendorPath,
-			SumVersion: pkg.SumVersion,
-			RelPath:    pkg.RelPath,
-			Fetch: &fetchInfo{
-				Type:   "git",
-				URL:    pkg.URL,
-				Rev:    pkg.Rev,
-				Sha256: pkg.Sha256,
-			},
-		}
-	}
-
-	var buf bytes.Buffer
-	e := toml.NewEncoder(&buf)
-	err := e.Encode(result)
-	if err != nil {
-		return nil, err
-	}
-
-	return buf.Bytes(), nil
-}
-
-func LoadGomod2Nix(filePath string) map[string]*types.Package {
-	ret := make(map[string]*types.Package)
-
-	if filePath == "" {
-		return ret
-	}
-
-	b, err := ioutil.ReadFile(filePath)
-	if err != nil {
-		fmt.Println(err)
-		return ret
-	}
-
-	result := make(map[string]*packageT)
-	_, err = toml.Decode(string(b), &result)
-	if err != nil {
-		fmt.Println(err)
-		return ret
-	}
-
-	for k, v := range result {
-		ret[k] = &types.Package{
-			GoPackagePath: k,
-			URL:           v.Fetch.URL,
-			Rev:           v.Fetch.Rev,
-			Sha256:        v.Fetch.Sha256,
-			SumVersion:    v.SumVersion,
-			RelPath:       v.RelPath,
-			VendorPath:    v.VendorPath,
-		}
-	}
-
-	return ret
-}

generate/generate.go

@@ -0,0 +1,144 @@
+package fetch
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"path/filepath"
+	"sort"
+	"sync"
+
+	"github.com/nix-community/go-nix/pkg/nar"
+	log "github.com/sirupsen/logrus"
+	"github.com/tweag/gomod2nix/lib"
+	schema "github.com/tweag/gomod2nix/schema"
+	"golang.org/x/mod/modfile"
+)
+
+type goModDownload struct {
+	Path     string
+	Version  string
+	Info     string
+	GoMod    string
+	Zip      string
+	Dir      string
+	Sum      string
+	GoModSum string
+}
+
+func GeneratePkgs(directory string, goMod2NixPath string) ([]*schema.Package, error) {
+	goModPath := filepath.Join(directory, "go.mod")
+
+	log.WithFields(log.Fields{
+		"modPath": goModPath,
+	}).Info("Parsing go.mod")
+
+	// Read go.mod
+	data, err := ioutil.ReadFile(goModPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse go.mod
+	mod, err := modfile.Parse(goModPath, data, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Map repos -> replacement repo
+	replace := make(map[string]string)
+	for _, repl := range mod.Replace {
+		replace[repl.New.Path] = repl.Old.Path
+	}
+
+	var modDownloads []*goModDownload
+	{
+		log.Info("Downloading dependencies")
+
+		cmd := exec.Command(
+			"go", "mod", "download", "--json",
+		)
+		cmd.Dir = directory
+		stdout, err := cmd.Output()
+		if err != nil {
+			return nil, err
+		}
+
+		dec := json.NewDecoder(bytes.NewReader(stdout))
+		for {
+			var dl *goModDownload
+			err := dec.Decode(&dl)
+			if err == io.EOF {
+				break
+			}
+			modDownloads = append(modDownloads, dl)
+		}
+
+		log.Info("Done downloading dependencies")
+	}
+
+	executor := lib.NewParallellExecutor()
+	var mux sync.Mutex
+
+	cache := schema.ReadCache(goMod2NixPath)
+
+	packages := []*schema.Package{}
+	addPkg := func(pkg *schema.Package) {
+		mux.Lock()
+		packages = append(packages, pkg)
+		mux.Unlock()
+	}
+
+	for _, dl := range modDownloads {
+		dl := dl
+
+		goPackagePath, hasReplace := replace[dl.Path]
+		if !hasReplace {
+			goPackagePath = dl.Path
+		}
+
+		cached, ok := cache[goPackagePath]
+		if ok && cached.Version == dl.Version {
+			addPkg(cached)
+			continue
+		}
+
+		executor.Add(func() error {
+			h := sha256.New()
+			err := nar.DumpPath(h, dl.Dir)
+			if err != nil {
+				return err
+			}
+			digest := h.Sum(nil)
+
+			pkg := &schema.Package{
+				GoPackagePath: goPackagePath,
+				Version:       dl.Version,
+				Hash:          "sha256-" + base64.StdEncoding.EncodeToString(digest),
+			}
+			if hasReplace {
+				pkg.ReplacedPath = dl.Path
+			}
+
+			addPkg(pkg)
+
+			return nil
+		})
+	}
+
+	err = executor.Wait()
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Slice(packages, func(i, j int) bool {
+		return packages[i].GoPackagePath < packages[j].GoPackagePath
+	})
+
+	return packages, nil
+
+}

go.mod

@@ -4,9 +4,8 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44
+	github.com/nix-community/go-nix v0.0.0-20220528121639-b940fb0a12d8
 	github.com/sirupsen/logrus v1.7.0
 	golang.org/x/mod v0.3.0
 	golang.org/x/sys v0.0.0-20220519141025-dcacdad47464 // indirect
-	golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
 )

go.sum

@@ -1,31 +1,30 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U=
+github.com/alecthomas/kong v0.5.0/go.mod h1:uzxf/HUh0tj43x1AyJROl3JT7SgsZ5m+icOv1csRhc0=
-github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI=
+github.com/alecthomas/participle/v2 v2.0.0-alpha7/go.mod h1:NumScqsC42o9x+dGj8/YqsIfhrIQjFEOFovxotbBirA=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721 h1:JHZL0hZKJ1VENNfmXvHbgYlbUOvpzYzvy2aZU5gXVeo=
+github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
-github.com/alecthomas/colour v0.0.0-20160524082231-60882d9e2721/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0=
+github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
-github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY=
-github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/orivej/e v0.0.0-20180728214217-ac3492690fda h1:fqLgbcmo9qKecZOH8lByuxi9XXoIhNYBpRJEo4rDEUQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/orivej/e v0.0.0-20180728214217-ac3492690fda/go.mod h1:eOxOguJBxQH6q/o7CZvmR+fh5v1LHH1sfohtgISSSFA=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44 h1:XDJpMiCKWt8CIT2LE1QrF4DdrvI1WciSNUrnYtNewPo=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44/go.mod h1:4SkaXpoQ0tQ0OIkGqU8ByPLANmTTTU1iWPDz7YXatSA=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
+github.com/nix-community/go-nix v0.0.0-20220528121639-b940fb0a12d8 h1:IYr+3pldQtTwS55Xm2+3Ry81MpSKNVpMyYMNr7yQpts=
+github.com/nix-community/go-nix v0.0.0-20220528121639-b940fb0a12d8/go.mod h1:r5pCQAjHNSDWTsy6+UbacPN8EzMVJiCAXDWbRN2qfwU=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
@@ -33,16 +32,21 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220519141025-dcacdad47464 h1:MpIuURY70f0iKp/oooEFtB2oENcHITo/z1b6u41pKCw=
 golang.org/x/sys v0.0.0-20220519141025-dcacdad47464/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

gomod2nix.toml

@@ -1,191 +1,81 @@
-["github.com/BurntSushi/toml"]
+schema = 1
-  sumVersion = "v0.3.1"
-  ["github.com/BurntSushi/toml".fetch]
-    type = "git"
-    url = "https://github.com/BurntSushi/toml"
-    rev = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
-    sha256 = "1fjdwwfzyzllgiwydknf1pwjvy49qxfsczqx5gz3y0izs7as99j6"
 
-["github.com/alecthomas/assert"]
+[mod]
-  sumVersion = "v0.0.0-20170929043011-405dbfeb8e38"
+  [mod."github.com/BurntSushi/toml"]
-  ["github.com/alecthomas/assert".fetch]
+    version = "v0.3.1"
-    type = "git"
+    hash = "sha256-Rqak1dE/Aj/+Kx1/pl3Hifgt+Q3OzuZ5fJR+/x3nTbo="
-    url = "https://github.com/alecthomas/assert"
+  [mod."github.com/alecthomas/kong"]
-    rev = "405dbfeb8e38effee6e723317226e93fff912d06"
+    version = "v0.5.0"
-    sha256 = "1l567pi17k593nrd1qlbmiq8z9jy3qs60px2a16fdpzjsizwqx8l"
+    hash = "sha256-6OtTXBnZSVvbdgkAfah2EuX20Fg/2v3JRSnKq1QoJbA="
-
+  [mod."github.com/alecthomas/participle/v2"]
-["github.com/alecthomas/colour"]
+    version = "v2.0.0-alpha7"
-  sumVersion = "v0.0.0-20160524082231-60882d9e2721"
+    hash = "sha256-Iy3/FxQ9kb0rp7N7olD/FuMNEja3OtPce61uGKjWReE="
-  ["github.com/alecthomas/colour".fetch]
+  [mod."github.com/alecthomas/repr"]
-    type = "git"
+    version = "v0.0.0-20210801044451-80ca428c5142"
-    url = "https://github.com/alecthomas/colour"
+    hash = "sha256-XUx+sN0wtc1j4t83sa5Z2MOBHxYQUUfsFfgfBeZqCUQ="
-    rev = "60882d9e27213e8552dcff6328914fe4c2b44bc9"
+  [mod."github.com/davecgh/go-spew"]
-    sha256 = "0iq566534gbzkd16ixg7fk298wd766821vvs80838yifx9yml5vs"
+    version = "v1.1.1"
-
+    hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
-["github.com/alecthomas/kingpin"]
+  [mod."github.com/google/go-cmp"]
-  sumVersion = "v2.2.6+incompatible"
+    version = "v0.5.5"
-  ["github.com/alecthomas/kingpin".fetch]
+    hash = "sha256-pyfTIu1fx6cGYHhm+yi8YP2TgERnnqKRjqSV7WGb1Yk="
-    type = "git"
+  [mod."github.com/kr/pretty"]
-    url = "https://github.com/alecthomas/kingpin"
+    version = "v0.1.0"
-    rev = "947dcec5ba9c011838740e680966fd7087a71d0d"
+    hash = "sha256-Fx+TaNrxW0VfzonT2jnd5MU09RRz7GJZkqAtJR6/pKI="
-    sha256 = "0mndnv3hdngr3bxp7yxfd47cas4prv98sqw534mx7vp38gd88n5r"
+  [mod."github.com/kr/pty"]
-
+    version = "v1.1.1"
-["github.com/alecthomas/repr"]
+    hash = "sha256-AVeS+ivwNzIrgWQaLtsmO2f2MYGpxIVqdac/EzaYI1Q="
-  sumVersion = "v0.0.0-20180818092828-117648cd9897"
+  [mod."github.com/kr/text"]
-  ["github.com/alecthomas/repr".fetch]
+    version = "v0.1.0"
-    type = "git"
+    hash = "sha256-QT65kTrNypS5GPWGvgnCpGLPlVbQAL4IYvuqAKhepb4="
-    url = "https://github.com/alecthomas/repr"
+  [mod."github.com/nix-community/go-nix"]
-    rev = "117648cd9897bc621d6d52cd05727b9abdd5528a"
+    version = "v0.0.0-20220528121639-b940fb0a12d8"
-    sha256 = "05v1rgzdqc8razf702laagrvhvx68xd9yxxmzd3dyz0d6425pdrp"
+    hash = "sha256-r6fq1mu0gRFqdI6msuAZp3ZoncAw5L16oBFPUA9vPYk="
-
+  [mod."github.com/pkg/errors"]
-["github.com/alecthomas/template"]
+    version = "v0.9.1"
-  sumVersion = "v0.0.0-20160405071501-a0175ee3bccc"
+    hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw="
-  ["github.com/alecthomas/template".fetch]
+  [mod."github.com/pmezard/go-difflib"]
-    type = "git"
+    version = "v1.0.0"
-    url = "https://github.com/alecthomas/template"
+    hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA="
-    rev = "a0175ee3bccc567396460bf5acd36800cb10c49c"
+  [mod."github.com/sirupsen/logrus"]
-    sha256 = "0qjgvvh26vk1cyfq9fadyhfgdj36f1iapbmr5xp6zqipldz8ffxj"
+    version = "v1.7.0"
-
+    hash = "sha256-VClDP4DJDdMFTMGWR7I8L4hQRMU7cOphLRyUfxCvnNY="
-["github.com/alecthomas/units"]
+  [mod."github.com/stretchr/objx"]
-  sumVersion = "v0.0.0-20151022065526-2efee857e7cf"
+    version = "v0.1.0"
-  ["github.com/alecthomas/units".fetch]
+    hash = "sha256-az0Vd4MG3JXfaYbj0Q6AOmNkrXgmXDeQm8+BBiDXmdA="
-    type = "git"
+  [mod."github.com/stretchr/testify"]
-    url = "https://github.com/alecthomas/units"
+    version = "v1.7.0"
-    rev = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a"
+    hash = "sha256-t1I9uCrn9vSUu/z5IZuNyGShmbOcJ6UGc2f75ZBrHzE="
-    sha256 = "1j65b91qb9sbrml9cpabfrcf07wmgzzghrl7809hjjhrmbzri5bl"
+  [mod."golang.org/x/crypto"]
-
+    version = "v0.0.0-20191011191535-87dc89f01550"
-["github.com/davecgh/go-spew"]
+    hash = "sha256-T0CaT5ri68osnd9VdNTZ0Ys+exaVbX4iC8MN60UNkXw="
-  sumVersion = "v1.1.1"
+  [mod."golang.org/x/mod"]
-  ["github.com/davecgh/go-spew".fetch]
+    version = "v0.3.0"
-    type = "git"
+    hash = "sha256-c+KKzeDUDxrKIx8un1DpETlTK+DmHP+zUjRZrgs0ejo="
-    url = "https://github.com/davecgh/go-spew"
+  [mod."golang.org/x/net"]
-    rev = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
+    version = "v0.0.0-20190620200207-3b0461eec859"
-    sha256 = "0hka6hmyvp701adzag2g26cxdj47g21x6jz4sc6jjz1mn59d474y"
+    hash = "sha256-lVNbp7zOBVbvzA1fFR4SkUskYfAMt2eILoWuDgk0oS4="
-
+  [mod."golang.org/x/sync"]
-["github.com/mattn/go-isatty"]
+    version = "v0.0.0-20190423024810-112230192c58"
-  sumVersion = "v0.0.3"
+    hash = "sha256-1lGQ6frW6gwuLzwomu530IsFh7vlQWQOphQ0IQeZIhY="
-  ["github.com/mattn/go-isatty".fetch]
+  [mod."golang.org/x/sys"]
-    type = "git"
+    version = "v0.0.0-20220519141025-dcacdad47464"
-    url = "https://github.com/mattn/go-isatty"
+    hash = "sha256-N4P3QkaeRf05dFMadS1IvhkNV4VS6cf/zkTBY1zwWv4="
-    rev = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  [mod."golang.org/x/text"]
-    sha256 = "06w45aqz2a6yrk25axbly2k5wmsccv8cspb94bfmz4izvw8h927n"
+    version = "v0.3.0"
-
+    hash = "sha256-0FFbaxF1ZuAQF3sCcA85e8MO6prFeHint36inija4NY="
-["github.com/orivej/e"]
+  [mod."golang.org/x/tools"]
-  sumVersion = "v0.0.0-20180728214217-ac3492690fda"
+    version = "v0.0.0-20191119224855-298f0cb1881e"
-  ["github.com/orivej/e".fetch]
+    hash = "sha256-xhp3MWK3dr+PJgfOGaFAURPdw/ar+GRb+wHBnCG8Tg0="
-    type = "git"
+  [mod."golang.org/x/xerrors"]
-    url = "https://github.com/orivej/e"
+    version = "v0.0.0-20191204190536-9bdfabe68543"
-    rev = "ac3492690fda471c5eab817efd86badf5c3ac158"
+    hash = "sha256-DCyizuMVp9AhKGv0vL/iRZz/R6jWFmQxDWj1NFeITvo="
-    sha256 = "11jizr28kfkr6zscjxg95pqi6cjp08aqnhs41sdhc98nww78ilkr"
+  [mod."gopkg.in/check.v1"]
-
+    version = "v1.0.0-20180628173108-788fd7840127"
-["github.com/orivej/go-nix"]
+    hash = "sha256-KsRJNTprd1UijnJusbHwQGM7Bdm45Jt/QL+cIUGNa2w="
-  sumVersion = "v0.0.0-20180830055821-dae45d921a44"
+  [mod."gopkg.in/yaml.v2"]
-  ["github.com/orivej/go-nix".fetch]
+    version = "v2.2.2"
-    type = "git"
+    hash = "sha256-SjNLmGSxoYabOi/zdjymW3mSgaxaS39btBQ3/aUIkgc="
-    url = "https://github.com/orivej/go-nix"
+  [mod."gopkg.in/yaml.v3"]
-    rev = "dae45d921a44e6ec05b4ea2b00d8e9efadf770c3"
+    version = "v3.0.0-20210107192922-496545a6307b"
-    sha256 = "17hfmsz8hs3h2d5c06j1bvbw8ijrhzm3iz911z5zydsl4x7y0cgy"
+    hash = "sha256-j8yDji+vqsitpRZirpb4w/Em8nstgf28wpwkcrOlxBk="
-
-["github.com/pkg/profile"]
-  sumVersion = "v1.2.1"
-  ["github.com/pkg/profile".fetch]
-    type = "git"
-    url = "https://github.com/pkg/profile"
-    rev = "5b67d428864e92711fcbd2f8629456121a56d91f"
-    sha256 = "0blqmvgqvdbqmh3fp9pfdxc9w1qfshrr0zy9whj0sn372bw64qnr"
-
-["github.com/pmezard/go-difflib"]
-  sumVersion = "v1.0.0"
-  ["github.com/pmezard/go-difflib".fetch]
-    type = "git"
-    url = "https://github.com/pmezard/go-difflib"
-    rev = "792786c7400a136282c1664665ae0a8db921c6c2"
-    sha256 = "0c1cn55m4rypmscgf0rrb88pn58j3ysvc2d0432dp3c6fqg6cnzw"
-
-["github.com/sergi/go-diff"]
-  sumVersion = "v1.0.0"
-  ["github.com/sergi/go-diff".fetch]
-    type = "git"
-    url = "https://github.com/sergi/go-diff"
-    rev = "1744e2970ca51c86172c8190fadad617561ed6e7"
-    sha256 = "0swiazj8wphs2zmk1qgq75xza6m19snif94h2m6fi8dqkwqdl7c7"
-
-["github.com/sirupsen/logrus"]
-  sumVersion = "v1.7.0"
-  ["github.com/sirupsen/logrus".fetch]
-    type = "git"
-    url = "https://github.com/sirupsen/logrus"
-    rev = "6699a89a232f3db797f2e280639854bbc4b89725"
-    sha256 = "1a59pw7zimvm8k423iq9l4f4qjj1ia1xc6pkmhwl2mxc46y2n442"
-
-["github.com/stretchr/testify"]
-  sumVersion = "v1.2.2"
-  ["github.com/stretchr/testify".fetch]
-    type = "git"
-    url = "https://github.com/stretchr/testify"
-    rev = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
-    sha256 = "0dlszlshlxbmmfxj5hlwgv3r22x0y1af45gn1vd198nvvs3pnvfs"
-
-["golang.org/x/crypto"]
-  sumVersion = "v0.0.0-20191011191535-87dc89f01550"
-  ["golang.org/x/crypto".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/crypto"
-    rev = "87dc89f01550277dc22b74ffcf4cd89fa2f40f4c"
-    sha256 = "0z4i1m2yn3f31ci7wvcm2rxkx2yiv7a78mfzklncmsz2k97rlh2g"
-
-["golang.org/x/mod"]
-  sumVersion = "v0.3.0"
-  ["golang.org/x/mod".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/mod"
-    rev = "859b3ef565e237f9f1a0fb6b55385c497545680d"
-    sha256 = "0ldgbx2zpprbsfn6p8pfgs4nn87gwbfcv2z0fa7n8alwsq2yw78q"
-
-["golang.org/x/net"]
-  sumVersion = "v0.0.0-20190620200207-3b0461eec859"
-  ["golang.org/x/net".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/net"
-    rev = "3b0461eec859c4b73bb64fdc8285971fd33e3938"
-    sha256 = "0l00c8l0a8xnv6qdpwfzxxsr58jggacgzdrwiprrfx2xqm37b6d5"
-
-["golang.org/x/sync"]
-  sumVersion = "v0.0.0-20190423024810-112230192c58"
-  ["golang.org/x/sync".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/sync"
-    rev = "112230192c580c3556b8cee6403af37a4fc5f28c"
-    sha256 = "05i2k43j2d0llq768hg5pf3hb2yhfzp9la1w5wp0rsnnzblr0lfn"
-
-["golang.org/x/sys"]
-  sumVersion = "v0.0.0-20191026070338-33540a1f6037"
-  ["golang.org/x/sys".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/sys"
-    rev = "33540a1f603772f9d4b761f416f5c10dade23e96"
-    sha256 = "0fjcv0vzvi6za0b4xmnk3932pr9f9gczzf03y0kgq3ry9rqg169y"
-
-["golang.org/x/text"]
-  sumVersion = "v0.3.0"
-  ["golang.org/x/text".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/text"
-    rev = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-    sha256 = "0r6x6zjzhr8ksqlpiwm5gdd7s209kwk5p4lw54xjvz10cs3qlq19"
-
-["golang.org/x/tools"]
-  sumVersion = "v0.0.0-20191119224855-298f0cb1881e"
-  ["golang.org/x/tools".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/tools"
-    rev = "298f0cb1881e71e68bc5bc17757fabe78a426460"
-    sha256 = "0l4pqwib2wnfnix6nvxl0yrq87ipi7ff1rz6z5bxfzy9dki1nzmv"
-
-["golang.org/x/xerrors"]
-  sumVersion = "v0.0.0-20191011141410-1b5146add898"
-  ["golang.org/x/xerrors".fetch]
-    type = "git"
-    url = "https://go.googlesource.com/xerrors"
-    rev = "1b5146add8981d58be77b16229c0ff0f8bebd8c1"
-    sha256 = "0w2akj91krxjag0xdhsg78470888nicc5ismc2ap9jqpss6v1zih"

lib/executor.go

@@ -0,0 +1,68 @@
+package lib
+
+import (
+	"sync"
+)
+
+// ParallellExecutor - Execute callback functions in parallell
+type ParallellExecutor struct {
+	errChan chan error
+	wg      *sync.WaitGroup
+	mux     *sync.Mutex
+
+	// Error returned by Wait(), cached for other Wait() invocations
+	err  error
+	done bool
+}
+
+func NewParallellExecutor() *ParallellExecutor {
+	return &ParallellExecutor{
+		errChan: make(chan error),
+		mux:     new(sync.Mutex),
+		wg:      new(sync.WaitGroup),
+
+		err:  nil,
+		done: false,
+	}
+}
+
+func (e *ParallellExecutor) Add(fn func() error) {
+	e.wg.Add(1)
+
+	go func() {
+		defer e.wg.Done()
+
+		err := fn()
+		if err != nil {
+			e.errChan <- err
+		}
+	}()
+}
+
+func (e *ParallellExecutor) Wait() error {
+	e.mux.Lock()
+	defer e.mux.Unlock()
+
+	if e.done {
+		return e.err
+	}
+
+	var err error
+
+	// Ensure channel is closed
+	go func() {
+		e.wg.Wait()
+		close(e.errChan)
+	}()
+
+	for err = range e.errChan {
+		if err != nil {
+			break
+		}
+	}
+
+	e.done = true
+	e.err = err
+
+	return err
+}

main.go

@@ -4,17 +4,15 @@ import (
 	"flag"
 	"fmt"
 	log "github.com/sirupsen/logrus"
-	"github.com/tweag/gomod2nix/fetch"
+	generate "github.com/tweag/gomod2nix/generate"
-	"github.com/tweag/gomod2nix/formats/gomod2nix"
+	schema "github.com/tweag/gomod2nix/schema"
 	"io/ioutil"
 	"path/filepath"
 )
 
 func main() {
 
-	var keepGoing = flag.Bool("keep-going", false, "Whether to panic or not if a rev cannot be resolved (default \"false\")")
 	var directory = flag.String("dir", "./", "Go project directory")
-	var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
 	var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
 	flag.Parse()
 
@@ -23,17 +21,14 @@ func main() {
 		outDir = *directory
 	}
 
-	goSumPath := filepath.Join(*directory, "go.sum")
-	goModPath := filepath.Join(*directory, "go.mod")
-
 	goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
 	outFile := goMod2NixPath
-	pkgs, err := fetch.FetchPackages(goModPath, goSumPath, goMod2NixPath, *maxJobs, *keepGoing)
+	pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath)
 	if err != nil {
 		panic(err)
 	}
 
-	output, err := gomod2nix.Marshal(pkgs)
+	output, err := schema.Marshal(pkgs)
 	if err != nil {
 		panic(err)
 	}

schema/schema.go

@@ -0,0 +1,71 @@
+package types
+
+import (
+	"bytes"
+	"github.com/BurntSushi/toml"
+	"os"
+)
+
+const SchemaVersion = 1
+
+type Package struct {
+	GoPackagePath string `toml:"-"`
+	Version       string `toml:"version"`
+	Hash          string `toml:"hash"`
+	ReplacedPath  string `toml:"replaced,omitempty"`
+}
+
+type Output struct {
+	SchemaVersion int                 `toml:"schema"`
+	Mod           map[string]*Package `toml:"mod"`
+}
+
+func Marshal(pkgs []*Package) ([]byte, error) {
+	out := &Output{
+		SchemaVersion: SchemaVersion,
+		Mod:           make(map[string]*Package),
+	}
+
+	for _, pkg := range pkgs {
+		out.Mod[pkg.GoPackagePath] = pkg
+	}
+
+	var buf bytes.Buffer
+	e := toml.NewEncoder(&buf)
+	err := e.Encode(out)
+	if err != nil {
+		return nil, err
+	}
+
+	return buf.Bytes(), nil
+}
+
+func ReadCache(filePath string) map[string]*Package {
+	ret := make(map[string]*Package)
+
+	if filePath == "" {
+		return ret
+	}
+
+	b, err := os.ReadFile(filePath)
+	if err != nil {
+		return ret
+	}
+
+	var output Output
+	_, err = toml.Decode(string(b), &output)
+	if err != nil {
+		return ret
+	}
+
+	if output.SchemaVersion != SchemaVersion {
+		return ret
+	}
+
+	for k, v := range output.Mod {
+		v.GoPackagePath = k
+		ret[k] = v
+	}
+
+	return ret
+}

shell.nix

@@ -16,16 +16,11 @@
   )
 }:
 
-let
-  pythonEnv = pkgs.python3.withPackages (_: [ ]);
-
-in
 pkgs.mkShell {
+  NIX_PATH = "nixpkgs=${builtins.toString pkgs.path}";
   buildInputs = [
-    pkgs.nix-prefetch-git
     pkgs.nixpkgs-fmt
-    pkgs.go
+    pkgs.gomod2nix.go
     pkgs.gomod2nix
-    pythonEnv
   ];
 }

tests/Makefile

@@ -1,2 +1,2 @@
 all:
-	python ./run.py
+	go run ./run.go

tests/helm/default.nix

@@ -23,11 +23,4 @@ buildGoApplication rec {
     $out/bin/helm completion zsh > helm.zsh
     installShellCompletion helm.{bash,zsh}
   '';
-
-  meta = with stdenv.lib; {
-    homepage = "https://github.com/kubernetes/helm";
-    description = "A package manager for kubernetes";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ rlupton20 edude03 saschagrunert Frostman Chili-Man ];
-  };
 }

tests/linkerd/default.nix

@@ -17,11 +17,4 @@ buildGoApplication {
   doCheck = false;
 
   subPackages = [ "cli/cmd" ];
-
-  meta = with stdenv.lib; {
-    description = "A service mesh for Kubernetes and beyond";
-    homepage = "https://linkerd.io/";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ Gonzih ];
-  };
 }

tests/minikube/default.nix

@@ -6,7 +6,7 @@
 , pkg-config
 , which
 , libvirt
-, vmnet
+, darwin
 }:
 
 buildGoApplication rec {
@@ -26,7 +26,7 @@ buildGoApplication rec {
 
   nativeBuildInputs = [ go-bindata installShellFiles pkg-config which ];
 
-  buildInputs = if stdenv.isDarwin then [ vmnet ] else if stdenv.isLinux then [ libvirt ] else null;
+  buildInputs = if stdenv.isDarwin then [ darwin.apple_sdk.frameworks.vmnet ] else if stdenv.isLinux then [ libvirt ] else null;
 
   buildPhase = ''
     make COMMIT=${src.rev}
@@ -45,11 +45,4 @@ buildGoApplication rec {
     done
   '';
 
-  meta = with stdenv.lib; {
-    homepage = "https://minikube.sigs.k8s.io";
-    description = "A tool that makes it easy to run Kubernetes locally";
-    license = licenses.asl20;
-    maintainers = with maintainers; [ ebzzry copumpkin vdemeester atkinschang Chili-Man ];
-    platforms = platforms.unix;
-  };
 }

tests/run.go

@@ -0,0 +1,197 @@
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"sync"
+)
+
+type testError struct {
+	testDir string
+	stdout  bytes.Buffer
+	stderr  bytes.Buffer
+}
+
+func runProcess(prefix string, command string, args ...string) error {
+	fmt.Println(fmt.Sprintf("%s: Executing %s %s", prefix, command, args))
+
+	cmd := exec.Command(command, args...)
+
+	stdoutReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	stderrReader, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	done := make(chan struct{})
+
+	go func() {
+		reader := io.MultiReader(stdoutReader, stderrReader)
+		scanner := bufio.NewScanner(reader)
+		for scanner.Scan() {
+			line := scanner.Bytes()
+			fmt.Println(fmt.Sprintf("%s: %s", prefix, line))
+		}
+		done <- struct{}{}
+	}()
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	<-done
+
+	return cmd.Wait()
+}
+
+func contains(haystack []string, needle string) bool {
+	for _, s := range haystack {
+		if s == needle {
+			return true
+		}
+	}
+	return false
+}
+
+func runTest(rootDir string, testDir string) error {
+	prefix := testDir
+	cmdPath := filepath.Join(rootDir, "..", "gomod2nix")
+	testDir = filepath.Join(rootDir, testDir)
+	err := runProcess(prefix, cmdPath, "--dir", testDir, "--outdir", testDir)
+	if err != nil {
+		return err
+	}
+
+	buildExpr := fmt.Sprintf("with (import <nixpkgs> { overlays = [ (import %s/../overlay.nix) ]; }); callPackage %s {}", rootDir, testDir)
+	err = runProcess(prefix, "nix-build", "--no-out-link", "--expr", buildExpr)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getTestDirs(rootDir string) ([]string, error) {
+	// Takes too long for Github Actions
+	var blacklist []string
+	if os.Getenv("GITHUB_ACTIONS") == "true" {
+		blacklist = []string{
+			"helm",
+			"minikube",
+		}
+	}
+
+	files, err := os.ReadDir(rootDir)
+	if err != nil {
+		return nil, err
+	}
+
+	testDirs := []string{}
+	for _, f := range files {
+		if f.IsDir() && !contains(blacklist, f.Name()) {
+			testDirs = append(testDirs, f.Name())
+		}
+	}
+
+	return testDirs, nil
+}
+
+func runTests(rootDir string, testDirs []string) error {
+	var wg sync.WaitGroup
+	cmdErrChan := make(chan error)
+	for _, testDir := range testDirs {
+		testDir := testDir
+		fmt.Println(fmt.Sprintf("Running test for: '%s'", testDir))
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := runTest(rootDir, testDir)
+			if err != nil {
+				cmdErrChan <- fmt.Errorf("Test for '%s' failed: %w", testDir, err)
+			}
+		}()
+	}
+
+	go func() {
+		wg.Wait()
+		close(cmdErrChan)
+	}()
+
+	for cmdErr := range cmdErrChan {
+		return cmdErr
+	}
+
+	return nil
+}
+
+func main() {
+
+	var rootDir string
+	{
+		_, file, _, ok := runtime.Caller(0)
+		if !ok {
+			panic("Couldn't get test directory")
+		}
+		rootDir = path.Dir(file)
+	}
+
+	flag.Parse()
+
+	nArgs := flag.NArg()
+
+	action := "run"
+	if nArgs >= 1 {
+		action = flag.Arg(0)
+	}
+
+	switch action {
+	case "list":
+		testDirs, err := getTestDirs(rootDir)
+		if err != nil {
+			panic(err)
+		}
+
+		for _, testDir := range testDirs {
+			fmt.Println(testDir)
+		}
+
+		return
+
+	case "run":
+		var testDirs []string
+		var err error
+		if nArgs > 1 {
+			args := flag.Args()
+			testDirs = args[1:nArgs]
+		} else {
+			testDirs, err = getTestDirs(rootDir)
+			if err != nil {
+				panic(err)
+			}
+		}
+
+		err = runTests(rootDir, testDirs)
+		if err != nil {
+			panic(err)
+		}
+
+		return
+
+	default:
+		panic(fmt.Errorf("Unhandled action: %s", flag.Arg(0)))
+	}
+
+}

tests/run.py

@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-import subprocess
-import os.path
-import sys
-
-
-if __name__ == '__main__':
-    script_dir = os.path.dirname(os.path.abspath(__file__))
-    root_dir = os.path.dirname(script_dir)
-
-    cmd = os.path.join(root_dir, "gomod2nix")
-
-    def run(directory):
-        print(f"Running {directory}")
-
-        subprocess.run([cmd, "--dir", directory, "--outdir", directory], check=True)
-
-        build_expr = ("""
-        with (import <nixpkgs> { overlays = [ (import %s/overlay.nix) ]; }); callPackage %s {}
-        """.replace("\n", " ") % (root_dir, directory))
-        subprocess.run(["nix-build", "--expr", build_expr], check=True)
-
-    for f in os.listdir(script_dir):
-
-        d = os.path.join(script_dir, f)
-        if os.path.isdir(d):
-            try:
-                run(d)
-            except Exception:
-                sys.stderr.write(f"Error running {d}\n")

types/types.go

@@ -1,11 +0,0 @@
-package types
-
-type Package struct {
-	GoPackagePath string
-	URL           string
-	Rev           string
-	Sha256        string
-	SumVersion    string
-	RelPath       string
-	VendorPath    string
-}