From 509b97c288e941c40ed5a1971465894668920af2 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 14 Jun 2022 06:32:19 +0800 Subject: [PATCH] Build non-development packages in the same way as development packages The difference in behaviour was unfortunate. The only downside I can see is that this doesn't allow for mixed origins in the same package, which probably shouldn't be done anyway. --- builder/default.nix | 109 +++++++++++++++++++++---------------- builder/symlink/symlink.go | 47 ++++++++-------- cmd/root.go | 29 +--------- generate/temp.go | 50 ++++++++++++----- gomod2nix.toml | 2 +- schema/schema.go | 8 +-- 6 files changed, 133 insertions(+), 112 deletions(-) diff --git a/builder/default.nix b/builder/default.nix index e55d6c6..ad9fcaf 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -12,6 +12,8 @@ }: let + inherit (builtins) substring; + parseGoMod = import ./parser.nix; removeExpr = refs: ''remove-references-to ${lib.concatMapStrings (ref: " -t ${ref}") refs}''; @@ -57,37 +59,43 @@ let impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "GOPROXY" ]; }; - mkVendorEnv = { go, modulesStruct, localReplaceCommands ? [ ] }: runCommand "vendor-env" - { - nativeBuildInputs = [ go ]; - json = builtins.toJSON modulesStruct; + mkVendorEnv = { go, modulesStruct, localReplaceCommands ? [ ], defaultPackage ? "" }: + let + sources = lib.mapAttrs + (goPackagePath: meta: fetchGoModule { + goPackagePath = meta.replaced or goPackagePath; + inherit (meta) version hash; + inherit go; + }) + modulesStruct.mod; + in + runCommand "vendor-env" + { + nativeBuildInputs = [ go ]; + json = builtins.toJSON (lib.filterAttrs (n: _: n != defaultPackage) modulesStruct.mod); - sources = builtins.toJSON ( - lib.mapAttrs - (goPackagePath: meta: fetchGoModule { - goPackagePath = meta.replaced or goPackagePath; - inherit (meta) version hash; - inherit go; - }) - modulesStruct.mod + sources = builtins.toJSON (lib.filterAttrs (n: _: n != defaultPackage) sources); + + passthru = { + inherit sources; + }; + + passAsFile = [ "json" "sources" ]; + } + ( + '' + mkdir vendor + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + + ${internal.symlink} + ${lib.concatStringsSep "\n" localReplaceCommands} + + mv vendor $out + '' ); - passAsFile = [ "json" "sources" ]; - } - ( - '' - mkdir vendor - - export GOCACHE=$TMPDIR/go-cache - export GOPATH="$TMPDIR/go" - - ${internal.symlink} - ${lib.concatStringsSep "\n" localReplaceCommands} - - mv vendor $out - '' - ); - # Select Go attribute based on version specified in go.mod selectGo = attrs: goMod: attrs.go or (if goMod == null then pkgs.go else ( @@ -101,6 +109,18 @@ let ) )); + # Strip the rubbish that Go adds to versions, and fall back to a version based on the date if it's a placeholder value + stripVersion = version: + let + parts = lib.elemAt (builtins.split "(\\+|-)" (lib.removePrefix "v" version)); + v = parts 0; + d = parts 2; + in + if v != "0.0.0" then v else "unstable-" + (lib.concatStringsSep "-" [ + (substring 0 4 d) + (substring 4 2 d) + (substring 6 2 d) + ]); mkGoEnv = { pwd @@ -165,9 +185,11 @@ let let modulesStruct = builtins.fromTOML (builtins.readFile modules); + goModPath = "${builtins.toString pwd}/go.mod"; + goMod = - if pwd != null - then parseGoMod (builtins.readFile "${builtins.toString pwd}/go.mod") + if pwd != null && lib.pathExists goModPath + then parseGoMod (builtins.readFile goModPath) else null; localReplaceCommands = let @@ -182,29 +204,30 @@ let )) localReplaceAttrs); in - if pwd != null then commands else [ ]; + if goMod != null then commands else [ ]; go = selectGo attrs goMod; removeReferences = [ ] ++ lib.optional (!allowGoReference) go; - vendorEnv = mkVendorEnv { - inherit go modulesStruct localReplaceCommands; - }; - defaultPackage = modulesStruct.goPackagePath or ""; + vendorEnv = mkVendorEnv { + inherit go modulesStruct localReplaceCommands defaultPackage; + }; + package = stdenv.mkDerivation (lib.optionalAttrs (defaultPackage != "") { pname = attrs.pname or baseNameOf defaultPackage; - version = lib.removePrefix "v" (modulesStruct.mod.${defaultPackage}).version; - } // attrs // { + version = stripVersion (modulesStruct.mod.${defaultPackage}).version; + src = vendorEnv.passthru.sources.${defaultPackage}; + } // lib.optionalAttrs (lib.hasAttr "subPackages" modulesStruct) { + subPackages = modulesStruct.subPackages; + } // attrs // { nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs; inherit (go) GOOS GOARCH; - inherit src; - GO_NO_VENDOR_CHECKS = "1"; GO111MODULE = "on"; @@ -312,12 +335,6 @@ let dir="$GOPATH/bin" [ -e "$dir" ] && cp -r $dir $out - ${lib.optionalString (lib.hasAttr "install" modulesStruct) '' - ${lib.concatStringsSep "\n" (map (x: "go install ${x}") (modulesStruct.install or [ ]))} - mkdir -p $out/bin - cp -a $GOPATH/bin/* $out/bin/ - ''} - runHook postInstall ''; @@ -329,7 +346,7 @@ let disallowedReferences = lib.optional (!allowGoReference) go; - passthru = passthru // { inherit go vendorEnv; }; + passthru = { inherit go vendorEnv; } // passthru; meta = { platforms = go.meta.platforms or lib.platforms.all; } // meta; }); diff --git a/builder/symlink/symlink.go b/builder/symlink/symlink.go index 2f22cf7..3dbb383 100644 --- a/builder/symlink/symlink.go +++ b/builder/symlink/symlink.go @@ -16,38 +16,41 @@ type Package struct { ReplacedPath string `json:"replaced,omitempty"` } -type Output struct { - SchemaVersion int `json:"schema"` - Mod map[string]*Package `json:"mod"` -} +// type Output struct { +// SchemaVersion int `json:"schema"` +// Mod map[string]*Package `json:"mod"` +// } func main() { - var output Output + // var output Output sources := make(map[string]string) + pkgs := make(map[string]*Package) - b, err := ioutil.ReadFile(os.Getenv("sourcesPath")) - if err != nil { - panic(err) + { + b, err := ioutil.ReadFile(os.Getenv("sourcesPath")) + if err != nil { + panic(err) + } + + err = json.Unmarshal(b, &sources) + if err != nil { + panic(err) + } } - err = json.Unmarshal(b, &sources) - if err != nil { - panic(err) - } + { + b, err := ioutil.ReadFile(os.Getenv("jsonPath")) + if err != nil { + panic(err) + } - b, err = ioutil.ReadFile(os.Getenv("jsonPath")) - if err != nil { - panic(err) + err = json.Unmarshal(b, &pkgs) + if err != nil { + panic(err) + } } - err = json.Unmarshal(b, &output) - if err != nil { - panic(err) - } - - pkgs := output.Mod - keys := make([]string, 0, len(pkgs)) for key := range pkgs { keys = append(keys, key) diff --git a/cmd/root.go b/cmd/root.go index d1b290c..99744d2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "io" "os" "path/filepath" @@ -69,14 +68,14 @@ func generateFunc(cmd *cobra.Command, args []string) { } var goPackagePath string - var install []string + var subPackages []string if tmpProj != nil { - install = tmpProj.Install + subPackages = tmpProj.SubPackages goPackagePath = tmpProj.GoPackagePath } - output, err := schema.Marshal(pkgs, goPackagePath, install) + output, err := schema.Marshal(pkgs, goPackagePath, subPackages) if err != nil { panic(fmt.Errorf("error marshaling output: %v", err)) } @@ -87,28 +86,6 @@ func generateFunc(cmd *cobra.Command, args []string) { } log.Info(fmt.Sprintf("Wrote: %s", outFile)) } - - // If we are dealing with a project packaged by passing packages on the command line, copy go.mod - if tmpProj != nil { - outMod := filepath.Join(outDir, "go.mod") - - fin, err := os.Open(filepath.Join(tmpProj.Dir, "go.mod")) - if err != nil { - panic(fmt.Errorf("error opening go.mod: %v", err)) - } - - fout, err := os.Create(outMod) - if err != nil { - panic(fmt.Errorf("error creating go.mod: %v", err)) - } - - _, err = io.Copy(fout, fin) - if err != nil { - panic(fmt.Errorf("error writing go.mod: %v", err)) - } - - log.Info(fmt.Sprintf("Wrote: %s", outMod)) - } } var rootCmd = &cobra.Command{ diff --git a/generate/temp.go b/generate/temp.go index cae2fa4..1af6a80 100644 --- a/generate/temp.go +++ b/generate/temp.go @@ -12,12 +12,13 @@ import ( "strings" log "github.com/sirupsen/logrus" + "golang.org/x/mod/module" "golang.org/x/tools/go/vcs" ) type TempProject struct { Dir string - Install []string + SubPackages []string GoPackagePath string } @@ -35,9 +36,7 @@ func NewTempProject(packages []string) (*TempProject, error) { var goPackagePath string - { - path := install[0] - + for _, path := range install { log.WithFields(log.Fields{ "path": path, }).Info("Finding repo root for import path") @@ -47,7 +46,15 @@ func NewTempProject(packages []string) (*TempProject, error) { return nil, err } - goPackagePath = repoRoot.Root + _, versionSuffix, _ := module.SplitPathVersion(path) + + p := repoRoot.Root + versionSuffix + + if goPackagePath != "" && p != goPackagePath { + return nil, fmt.Errorf("Mixed origin packages are not allowed") + } + + goPackagePath = p } log.Info("Setting up temporary project") @@ -132,31 +139,48 @@ func NewTempProject(packages []string) (*TempProject, error) { }).Info("Done initializing go.mod") // For every dependency fetch it - for _, imp := range packages { + { log.WithFields(log.Fields{ "dir": dir, - "dep": imp, - }).Info("Getting dependency") + }).Info("Getting dependencies") - cmd := exec.Command("go", "get", "-d", imp) + args := []string{"get", "-d"} + args = append(args, packages...) + + cmd := exec.Command("go", args...) cmd.Dir = dir cmd.Stderr = os.Stderr _, err := cmd.Output() if err != nil { - return nil, fmt.Errorf("error fetching '%s': %v", imp, err) + return nil, fmt.Errorf("error fetching: %v", err) } log.WithFields(log.Fields{ "dir": dir, - "dep": imp, - }).Info("Done getting dependency") + }).Info("Done getting dependencies") + } + } + + subPackages := []string{} + { + prefix, versionSuffix, _ := module.SplitPathVersion(goPackagePath) + for _, path := range install { + p := strings.TrimPrefix(path, prefix) + p = strings.TrimSuffix(p, versionSuffix) + p = strings.TrimPrefix(p, "/") + + if p == "" { + continue + } + + subPackages = append(subPackages, p) } } return &TempProject{ Dir: dir, - Install: install, + SubPackages: subPackages, GoPackagePath: goPackagePath, }, nil } diff --git a/gomod2nix.toml b/gomod2nix.toml index e99de42..1ab7844 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -1,4 +1,4 @@ -schema = 2 +schema = 3 [mod] [mod."cloud.google.com/go"] diff --git a/schema/schema.go b/schema/schema.go index 0c5d497..2ca359f 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -6,7 +6,7 @@ import ( "os" ) -const SchemaVersion = 2 +const SchemaVersion = 3 type Package struct { GoPackagePath string `toml:"-"` @@ -20,17 +20,17 @@ type Output struct { Mod map[string]*Package `toml:"mod"` // Packages with passed import paths trigger `go install` based on this list - Install []string `toml:"install,omitempty"` + SubPackages []string `toml:"subPackages,omitempty"` // Packages with passed import paths has a "default package" which pname & version is inherit from GoPackagePath string `toml:"goPackagePath,omitempty"` } -func Marshal(pkgs []*Package, goPackagePath string, install []string) ([]byte, error) { +func Marshal(pkgs []*Package, goPackagePath string, subPackages []string) ([]byte, error) { out := &Output{ SchemaVersion: SchemaVersion, Mod: make(map[string]*Package), - Install: install, + SubPackages: subPackages, GoPackagePath: goPackagePath, }