Build non-development packages in the same way as development packages

The difference in behaviour was unfortunate.

The only downside I can see is that this doesn't allow for mixed
origins in the same package, which probably shouldn't be done anyway.

builder/default.nix

@@ -12,6 +12,8 @@
 }:
 let
 
+  inherit (builtins) substring;
+
   parseGoMod = import ./parser.nix;
 
   removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}'';
@@ -57,37 +59,43 @@ let
       impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ];
     };
 
-  mkVendorEnv = { go, modulesStruct, localReplaceCommands ? [ ] }: runCommand "vendor-env"
+  mkVendorEnv = { go, modulesStruct, localReplaceCommands ? [ ], defaultPackage ? "" }:
-    {
+    let
-      nativeBuildInputs = [ go ];
+      sources = lib.mapAttrs
-      json = builtins.toJSON modulesStruct;
+        (goPackagePath: meta: fetchGoModule {
+          goPackagePath = meta.replaced or goPackagePath;
+          inherit (meta) version hash;
+          inherit go;
+        })
+        modulesStruct.mod;
+    in
+    runCommand "vendor-env"
+      {
+        nativeBuildInputs = [ go ];
+        json = builtins.toJSON (lib.filterAttrs (n: _: n != defaultPackage) modulesStruct.mod);
 
-      sources = builtins.toJSON (
+        sources = builtins.toJSON (lib.filterAttrs (n: _: n != defaultPackage) sources);
-        lib.mapAttrs
+
-          (goPackagePath: meta: fetchGoModule {
+        passthru = {
-            goPackagePath = meta.replaced or goPackagePath;
+          inherit sources;
-            inherit (meta) version hash;
+        };
-            inherit go;
+
-          })
+        passAsFile = [ "json" "sources" ];
-          modulesStruct.mod
+      }
+      (
+        ''
+          mkdir vendor
+
+          export GOCACHE=$TMPDIR/go-cache
+          export GOPATH="$TMPDIR/go"
+
+          ${internal.symlink}
+          ${lib.concatStringsSep "\n" localReplaceCommands}
+
+          mv vendor $out
+        ''
       );
 
-      passAsFile = [ "json" "sources" ];
-    }
-    (
-      ''
-        mkdir vendor
-
-        export GOCACHE=$TMPDIR/go-cache
-        export GOPATH="$TMPDIR/go"
-
-        ${internal.symlink}
-        ${lib.concatStringsSep "\n" localReplaceCommands}
-
-        mv vendor $out
-      ''
-    );
-
   # Select Go attribute based on version specified in go.mod
   selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else
   (
@@ -101,6 +109,18 @@ let
     )
   ));
 
+  # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value
+  stripVersion = version:
+    let
+      parts = lib.elemAt (builtins.split "(\\+|-)" (lib.removePrefix "v" version));
+      v = parts 0;
+      d = parts 2;
+    in
+    if v != "0.0.0" then v else "unstable-" + (lib.concatStringsSep "-" [
+      (substring 0 4 d)
+      (substring 4 2 d)
+      (substring 6 2 d)
+    ]);
 
   mkGoEnv =
     { pwd
@@ -165,9 +185,11 @@ let
     let
       modulesStruct = builtins.fromTOML (builtins.readFile modules);
 
+      goModPath = "${builtins.toString pwd}/go.mod";
+
       goMod =
-        if pwd != null
+        if pwd != null && lib.pathExists goModPath
-        then parseGoMod (builtins.readFile "${builtins.toString pwd}/go.mod")
+        then parseGoMod (builtins.readFile goModPath)
         else null;
       localReplaceCommands =
         let
@@ -182,29 +204,30 @@ let
               ))
               localReplaceAttrs);
         in
-        if pwd != null then commands else [ ];
+        if goMod != null then commands else [ ];
 
       go = selectGo attrs goMod;
 
       removeReferences = [ ] ++ lib.optional (!allowGoReference) go;
 
-      vendorEnv = mkVendorEnv {
-        inherit go modulesStruct localReplaceCommands;
-      };
-
       defaultPackage = modulesStruct.goPackagePath or "";
 
+      vendorEnv = mkVendorEnv {
+        inherit go modulesStruct localReplaceCommands defaultPackage;
+      };
+
       package = stdenv.mkDerivation (lib.optionalAttrs (defaultPackage != "")
         {
           pname = attrs.pname or baseNameOf defaultPackage;
-          version = lib.removePrefix "v" (modulesStruct.mod.${defaultPackage}).version;
+          version = stripVersion (modulesStruct.mod.${defaultPackage}).version;
-        } // attrs // {
+          src = vendorEnv.passthru.sources.${defaultPackage};
+        } // lib.optionalAttrs (lib.hasAttr "subPackages" modulesStruct) {
+        subPackages = modulesStruct.subPackages;
+      } // attrs // {
         nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
 
         inherit (go) GOOS GOARCH;
 
-        inherit src;
-
         GO_NO_VENDOR_CHECKS = "1";
 
         GO111MODULE = "on";
@@ -312,12 +335,6 @@ let
           dir="$GOPATH/bin"
           [ -e "$dir" ] && cp -r $dir $out
 
-          ${lib.optionalString (lib.hasAttr "install" modulesStruct) ''
-            ${lib.concatStringsSep "\n" (map (x: "go install ${x}") (modulesStruct.install or [ ]))}
-            mkdir -p $out/bin
-            cp -a $GOPATH/bin/* $out/bin/
-          ''}
-
           runHook postInstall
         '';
 
@@ -329,7 +346,7 @@ let
 
         disallowedReferences = lib.optional (!allowGoReference) go;
 
-        passthru = passthru // { inherit go vendorEnv; };
+        passthru = { inherit go vendorEnv; } // passthru;
 
         meta = { platforms = go.meta.platforms or lib.platforms.all; } // meta;
       });

builder/symlink/symlink.go

@@ -16,38 +16,41 @@ type Package struct {
 	ReplacedPath  string `json:"replaced,omitempty"`
 }
 
-type Output struct {
+// type Output struct {
-	SchemaVersion int                 `json:"schema"`
+// 	SchemaVersion int                 `json:"schema"`
-	Mod           map[string]*Package `json:"mod"`
+// 	Mod           map[string]*Package `json:"mod"`
-}
+// }
 
 func main() {
 
-	var output Output
+	// var output Output
 	sources := make(map[string]string)
+	pkgs := make(map[string]*Package)
 
-	b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
+	{
-	if err != nil {
+		b, err := ioutil.ReadFile(os.Getenv("sourcesPath"))
-		panic(err)
+		if err != nil {
+			panic(err)
+		}
+
+		err = json.Unmarshal(b, &sources)
+		if err != nil {
+			panic(err)
+		}
 	}
 
-	err = json.Unmarshal(b, &sources)
+	{
-	if err != nil {
+		b, err := ioutil.ReadFile(os.Getenv("jsonPath"))
-		panic(err)
+		if err != nil {
-	}
+			panic(err)
+		}
 
-	b, err = ioutil.ReadFile(os.Getenv("jsonPath"))
+		err = json.Unmarshal(b, &pkgs)
-	if err != nil {
+		if err != nil {
-		panic(err)
+			panic(err)
+		}
 	}
 
-	err = json.Unmarshal(b, &output)
-	if err != nil {
-		panic(err)
-	}
-
-	pkgs := output.Mod
-
 	keys := make([]string, 0, len(pkgs))
 	for key := range pkgs {
 		keys = append(keys, key)

cmd/root.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"path/filepath"
 
@@ -69,14 +68,14 @@ func generateFunc(cmd *cobra.Command, args []string) {
 		}
 
 		var goPackagePath string
-		var install []string
+		var subPackages []string
 
 		if tmpProj != nil {
-			install = tmpProj.Install
+			subPackages = tmpProj.SubPackages
 			goPackagePath = tmpProj.GoPackagePath
 		}
 
-		output, err := schema.Marshal(pkgs, goPackagePath, install)
+		output, err := schema.Marshal(pkgs, goPackagePath, subPackages)
 		if err != nil {
 			panic(fmt.Errorf("error marshaling output: %v", err))
 		}
@@ -87,28 +86,6 @@ func generateFunc(cmd *cobra.Command, args []string) {
 		}
 		log.Info(fmt.Sprintf("Wrote: %s", outFile))
 	}
-
-	// If we are dealing with a project packaged by passing packages on the command line, copy go.mod
-	if tmpProj != nil {
-		outMod := filepath.Join(outDir, "go.mod")
-
-		fin, err := os.Open(filepath.Join(tmpProj.Dir, "go.mod"))
-		if err != nil {
-			panic(fmt.Errorf("error opening go.mod: %v", err))
-		}
-
-		fout, err := os.Create(outMod)
-		if err != nil {
-			panic(fmt.Errorf("error creating go.mod: %v", err))
-		}
-
-		_, err = io.Copy(fout, fin)
-		if err != nil {
-			panic(fmt.Errorf("error writing go.mod: %v", err))
-		}
-
-		log.Info(fmt.Sprintf("Wrote: %s", outMod))
-	}
 }
 
 var rootCmd = &cobra.Command{

generate/temp.go

@@ -12,12 +12,13 @@ import (
 	"strings"
 
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/module"
 	"golang.org/x/tools/go/vcs"
 )
 
 type TempProject struct {
 	Dir           string
-	Install       []string
+	SubPackages   []string
 	GoPackagePath string
 }
 
@@ -35,9 +36,7 @@ func NewTempProject(packages []string) (*TempProject, error) {
 
 	var goPackagePath string
 
-	{
+	for _, path := range install {
-		path := install[0]
-
 		log.WithFields(log.Fields{
 			"path": path,
 		}).Info("Finding repo root for import path")
@@ -47,7 +46,15 @@ func NewTempProject(packages []string) (*TempProject, error) {
 			return nil, err
 		}
 
-		goPackagePath = repoRoot.Root
+		_, versionSuffix, _ := module.SplitPathVersion(path)
+
+		p := repoRoot.Root + versionSuffix
+
+		if goPackagePath != "" && p != goPackagePath {
+			return nil, fmt.Errorf("Mixed origin packages are not allowed")
+		}
+
+		goPackagePath = p
 	}
 
 	log.Info("Setting up temporary project")
@@ -132,31 +139,48 @@ func NewTempProject(packages []string) (*TempProject, error) {
 		}).Info("Done initializing go.mod")
 
 		// For every dependency fetch it
-		for _, imp := range packages {
+		{
 			log.WithFields(log.Fields{
 				"dir": dir,
-				"dep": imp,
+			}).Info("Getting dependencies")
-			}).Info("Getting dependency")
 
-			cmd := exec.Command("go", "get", "-d", imp)
+			args := []string{"get", "-d"}
+			args = append(args, packages...)
+
+			cmd := exec.Command("go", args...)
 			cmd.Dir = dir
 			cmd.Stderr = os.Stderr
 
 			_, err := cmd.Output()
 			if err != nil {
-				return nil, fmt.Errorf("error fetching '%s': %v", imp, err)
+				return nil, fmt.Errorf("error fetching: %v", err)
 			}
 
 			log.WithFields(log.Fields{
 				"dir": dir,
-				"dep": imp,
+			}).Info("Done getting dependencies")
-			}).Info("Done getting dependency")
+		}
+	}
+
+	subPackages := []string{}
+	{
+		prefix, versionSuffix, _ := module.SplitPathVersion(goPackagePath)
+		for _, path := range install {
+			p := strings.TrimPrefix(path, prefix)
+			p = strings.TrimSuffix(p, versionSuffix)
+			p = strings.TrimPrefix(p, "/")
+
+			if p == "" {
+				continue
+			}
+
+			subPackages = append(subPackages, p)
 		}
 	}
 
 	return &TempProject{
 		Dir:           dir,
-		Install:       install,
+		SubPackages:   subPackages,
 		GoPackagePath: goPackagePath,
 	}, nil
 }

gomod2nix.toml

@@ -1,4 +1,4 @@
-schema = 2
+schema = 3
 
 [mod]
   [mod."cloud.google.com/go"]

schema/schema.go

@@ -6,7 +6,7 @@ import (
 	"os"
 )
 
-const SchemaVersion = 2
+const SchemaVersion = 3
 
 type Package struct {
 	GoPackagePath string `toml:"-"`
@@ -20,17 +20,17 @@ type Output struct {
 	Mod           map[string]*Package `toml:"mod"`
 
 	// Packages with passed import paths trigger `go install` based on this list
-	Install []string `toml:"install,omitempty"`
+	SubPackages []string `toml:"subPackages,omitempty"`
 
 	// Packages with passed import paths has a "default package" which pname & version is inherit from
 	GoPackagePath string `toml:"goPackagePath,omitempty"`
 }
 
-func Marshal(pkgs []*Package, goPackagePath string, install []string) ([]byte, error) {
+func Marshal(pkgs []*Package, goPackagePath string, subPackages []string) ([]byte, error) {
 	out := &Output{
 		SchemaVersion: SchemaVersion,
 		Mod:           make(map[string]*Package),
-		Install:       install,
+		SubPackages:   subPackages,
 		GoPackagePath: goPackagePath,
 	}