From 1ffea526a0b416bf22f0796193437214b50c916f Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 14 Jun 2022 03:26:50 +0800 Subject: [PATCH] Add package generation for non development packages This makes it possible to generate packages that you do not have a local checkout for, e.g. running: `gomod2nix generate --outdir example/ golang.org/x/tools/cmd/stringer` This will be useful for packaging dependencies that you are not developing, but just simply packaging. --- .gitignore | 1 + builder/default.nix | 20 ++++- cmd/root.go | 168 +++++++++++++++++++++++++++---------- generate/generate.go | 2 +- generate/temp.go | 166 ++++++++++++++++++++++++++++++++++++ go.mod | 3 +- go.sum | 5 ++ gomod2nix.toml | 2 +- schema/schema.go | 12 ++- tests/cli-args/default.nix | 17 ++++ tests/cli-args/script | 4 + tests/run.go | 23 +++-- 12 files changed, 368 insertions(+), 55 deletions(-) create mode 100644 generate/temp.go create mode 100644 tests/cli-args/default.nix create mode 100755 tests/cli-args/script diff --git a/.gitignore b/.gitignore index ecd297f..6503950 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /gomod2nix result* /tests/*/gomod2nix.toml +/tests/*/go.mod diff --git a/builder/default.nix b/builder/default.nix index 23ff061..5131fd3 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -151,8 +151,8 @@ let }; buildGoApplication = - { modules - , src + { modules ? pwd + "/gomod2nix.toml" + , src ? pwd , pwd ? null , nativeBuildInputs ? [ ] , allowGoReference ? false @@ -190,11 +190,19 @@ let inherit go modulesStruct localReplaceCommands; }; - package = stdenv.mkDerivation (attrs // { + defaultPackage = modulesStruct.goPackagePath or ""; + + package = stdenv.mkDerivation (lib.optionalAttrs (defaultPackage != "") + { + pname = attrs.pname or baseNameOf defaultPackage; + version = lib.removePrefix "v" (modulesStruct.mod.${defaultPackage}).version; + } // attrs // { nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs; inherit (go) GOOS GOARCH; + inherit src; + GO_NO_VENDOR_CHECKS = "1"; GO111MODULE = "on"; @@ -302,6 +310,12 @@ let dir="$GOPATH/bin" [ -e "$dir" ] && cp -r $dir $out + ${lib.optionalString (lib.hasAttr "install" modulesStruct) '' + ${lib.concatStringsSep "\n" (map (x: "go install ${x}") (modulesStruct.install or [ ]))} + mkdir -p $out/bin + cp -a $GOPATH/bin/* $out/bin/ + ''} + runHook postInstall ''; diff --git a/cmd/root.go b/cmd/root.go index ceacbc1..d1b290c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "io" "os" "path/filepath" @@ -11,39 +12,148 @@ import ( schema "github.com/tweag/gomod2nix/schema" ) +const directoryDefault = "./" + var ( flagDirectory string flagOutDir string - flagMaxJobs int + maxJobs int ) +func generateFunc(cmd *cobra.Command, args []string) { + directory := flagDirectory + outDir := flagOutDir + + // If we are dealing with a project packaged by passing packages on the command line + // we need to create a temporary project. + var tmpProj *generate.TempProject + if len(args) > 0 { + var err error + + if directory != directoryDefault { + panic(fmt.Errorf("directory flag not supported together with import arguments")) + } + if outDir == "" { + pwd, err := os.Getwd() + if err != nil { + panic(err) + } + + outDir = pwd + } + + tmpProj, err = generate.NewTempProject(args) + if err != nil { + panic(err) + } + defer func() { + err := tmpProj.Remove() + if err != nil { + panic(err) + } + }() + + directory = tmpProj.Dir + } else if outDir == "" { + // Default out to current working directory if we are developing some software in the current repo. + outDir = directory + } + + // Write gomod2nix.toml + { + goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml") + outFile := goMod2NixPath + pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs) + if err != nil { + panic(fmt.Errorf("error generating pkgs: %v", err)) + } + + var goPackagePath string + var install []string + + if tmpProj != nil { + install = tmpProj.Install + goPackagePath = tmpProj.GoPackagePath + } + + output, err := schema.Marshal(pkgs, goPackagePath, install) + if err != nil { + panic(fmt.Errorf("error marshaling output: %v", err)) + } + + err = os.WriteFile(outFile, output, 0644) + if err != nil { + panic(fmt.Errorf("error writing file: %v", err)) + } + log.Info(fmt.Sprintf("Wrote: %s", outFile)) + } + + // If we are dealing with a project packaged by passing packages on the command line, copy go.mod + if tmpProj != nil { + outMod := filepath.Join(outDir, "go.mod") + + fin, err := os.Open(filepath.Join(tmpProj.Dir, "go.mod")) + if err != nil { + panic(fmt.Errorf("error opening go.mod: %v", err)) + } + + fout, err := os.Create(outMod) + if err != nil { + panic(fmt.Errorf("error creating go.mod: %v", err)) + } + + _, err = io.Copy(fout, fin) + if err != nil { + panic(fmt.Errorf("error writing go.mod: %v", err)) + } + + log.Info(fmt.Sprintf("Wrote: %s", outMod)) + } +} + var rootCmd = &cobra.Command{ Use: "gomod2nix", Short: "Convert applications using Go modules -> Nix", - Run: func(cmd *cobra.Command, args []string) { - err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs) - if err != nil { - panic(err) - } - }, + Run: generateFunc, } var generateCmd = &cobra.Command{ Use: "generate", Short: "Run gomod2nix.toml generator", - Run: func(cmd *cobra.Command, args []string) { - err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs) - if err != nil { - panic(err) - } - }, + Run: generateFunc, + + // func(cmd *cobra.Command, args []string) error { + // outDir := flagOutDir + // dir := flagDirectory + + // if len(args) > 0 { + // tmpProj, err := generate.NewTempProject(args) + // if err != nil { + // panic(err) + // } + // defer tmpProj.Remove() + + // dir = tmpProj.Dir + // outDir = "./tmp" + // } + + // // dir := "/home/adisbladis/go/pkg/mod/github.com/kyleconroy/sqlc@v1.14.0" + // // fmt.Println(args) + + // err := generateInternal(dir, outDir, flagMaxJobs) + // if err != nil { + // panic(err) + // } + + // return nil + // }, } var importCmd = &cobra.Command{ Use: "import", Short: "Import Go sources into the Nix store", Run: func(cmd *cobra.Command, args []string) { - err := generate.ImportPkgs(flagDirectory, flagMaxJobs) + err := generate.ImportPkgs(flagDirectory, maxJobs) if err != nil { panic(err) } @@ -51,9 +161,9 @@ var importCmd = &cobra.Command{ } func init() { - rootCmd.PersistentFlags().StringVar(&flagDirectory, "dir", "", "Go project directory") + rootCmd.PersistentFlags().StringVar(&flagDirectory, "dir", "./", "Go project directory") rootCmd.PersistentFlags().StringVar(&flagOutDir, "outdir", "", "Output directory (defaults to project directory)") - rootCmd.PersistentFlags().IntVar(&flagMaxJobs, "jobs", 10, "Max number of concurrent jobs") + rootCmd.PersistentFlags().IntVar(&maxJobs, "jobs", 10, "Max number of concurrent jobs") rootCmd.AddCommand(generateCmd) rootCmd.AddCommand(importCmd) @@ -65,29 +175,3 @@ func Execute() { os.Exit(1) } } - -func generateInternal(directory string, outDir string, maxJobs int) error { - if outDir == "" { - outDir = directory - } - - goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml") - outFile := goMod2NixPath - pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs) - if err != nil { - return fmt.Errorf("error generating pkgs: %v", err) - } - - output, err := schema.Marshal(pkgs) - if err != nil { - return fmt.Errorf("error marshaling output: %v", err) - } - - err = os.WriteFile(outFile, output, 0644) - if err != nil { - return fmt.Errorf("error writing file: %v", err) - } - log.Info(fmt.Sprintf("Wrote: %s", outFile)) - - return nil -} diff --git a/generate/generate.go b/generate/generate.go index cf95ce7..acdeed5 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -1,4 +1,4 @@ -package fetch +package generate import ( "bytes" diff --git a/generate/temp.go b/generate/temp.go new file mode 100644 index 0000000..cae2fa4 --- /dev/null +++ b/generate/temp.go @@ -0,0 +1,166 @@ +package generate + +import ( + "fmt" + "go/ast" + "go/printer" + "go/token" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" + "golang.org/x/tools/go/vcs" +) + +type TempProject struct { + Dir string + Install []string + GoPackagePath string +} + +func NewTempProject(packages []string) (*TempProject, error) { + // Imports without version suffix + install := make([]string, len(packages)) + for i, imp := range packages { + idx := strings.Index(imp, "@") + if idx == -1 { + idx = len(imp) + } + + install[i] = imp[:idx] + } + + var goPackagePath string + + { + path := install[0] + + log.WithFields(log.Fields{ + "path": path, + }).Info("Finding repo root for import path") + + repoRoot, err := vcs.RepoRootForImportPath(path, false) + if err != nil { + return nil, err + } + + goPackagePath = repoRoot.Root + } + + log.Info("Setting up temporary project") + + dir, err := os.MkdirTemp("", "gomod2nix-proj") + if err != nil { + return nil, err + } + + log.WithFields(log.Fields{ + "dir": dir, + }).Info("Created temporary directory") + + // Create tools.go + { + log.WithFields(log.Fields{ + "dir": dir, + }).Info("Creating tools.go") + + astFile := &ast.File{ + Name: ast.NewIdent("main"), + Decls: []ast.Decl{ + &ast.GenDecl{ + Tok: token.IMPORT, + Specs: func() []ast.Spec { + specs := make([]ast.Spec, len(install)) + + i := 0 + for _, imp := range install { + specs[i] = &ast.ImportSpec{ + Name: ast.NewIdent("_"), + Path: &ast.BasicLit{ + ValuePos: token.NoPos, + Kind: token.STRING, + Value: strconv.Quote(imp), + }, + } + + i++ + } + + return specs + }(), + }, + }, + } + + f, err := os.Create(filepath.Join(dir, "tools.go")) + if err != nil { + return nil, fmt.Errorf("Error creating tools.go: %v", err) + } + defer f.Close() + + fset := token.NewFileSet() + err = printer.Fprint(f, fset, astFile) + if err != nil { + return nil, fmt.Errorf("error writing tools.go: %v", err) + } + + log.WithFields(log.Fields{ + "dir": dir, + }).Info("Created tools.go") + } + + // Set up go module + { + log.WithFields(log.Fields{ + "dir": dir, + }).Info("Initializing go.mod") + + cmd := exec.Command("go", "mod", "init", "gomod2nix/dummy/package") + cmd.Dir = dir + cmd.Stderr = os.Stderr + + _, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("error creating go module: %v", err) + } + + log.WithFields(log.Fields{ + "dir": dir, + }).Info("Done initializing go.mod") + + // For every dependency fetch it + for _, imp := range packages { + log.WithFields(log.Fields{ + "dir": dir, + "dep": imp, + }).Info("Getting dependency") + + cmd := exec.Command("go", "get", "-d", imp) + cmd.Dir = dir + cmd.Stderr = os.Stderr + + _, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("error fetching '%s': %v", imp, err) + } + + log.WithFields(log.Fields{ + "dir": dir, + "dep": imp, + }).Info("Done getting dependency") + } + } + + return &TempProject{ + Dir: dir, + Install: install, + GoPackagePath: goPackagePath, + }, nil +} + +func (t *TempProject) Remove() error { + return os.RemoveAll(t.Dir) +} diff --git a/go.mod b/go.mod index bd221ff..b6b78c4 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,13 @@ require ( github.com/BurntSushi/toml v1.1.0 github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47 github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.4.0 golang.org/x/mod v0.5.1 + golang.org/x/tools v0.0.0-20210106214847-113979e3529a ) require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/spf13/cobra v1.4.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect diff --git a/go.sum b/go.sum index 98624d7..74f79fb 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= @@ -48,6 +49,7 @@ github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47 h1:K270nNzKXB github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47/go.mod h1:eE1NFc/GHPnxAhTTuxIfB6JSvRQdMVQCMQqRrGRWWfQ= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -68,6 +70,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -119,6 +122,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -137,5 +141,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/gomod2nix.toml b/gomod2nix.toml index 0124bff..e99de42 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -1,4 +1,4 @@ -schema = 1 +schema = 2 [mod] [mod."cloud.google.com/go"] diff --git a/schema/schema.go b/schema/schema.go index 652a7f4..0c5d497 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -6,7 +6,7 @@ import ( "os" ) -const SchemaVersion = 1 +const SchemaVersion = 2 type Package struct { GoPackagePath string `toml:"-"` @@ -18,12 +18,20 @@ type Package struct { type Output struct { SchemaVersion int `toml:"schema"` Mod map[string]*Package `toml:"mod"` + + // Packages with passed import paths trigger `go install` based on this list + Install []string `toml:"install,omitempty"` + + // Packages with passed import paths has a "default package" which pname & version is inherit from + GoPackagePath string `toml:"goPackagePath,omitempty"` } -func Marshal(pkgs []*Package) ([]byte, error) { +func Marshal(pkgs []*Package, goPackagePath string, install []string) ([]byte, error) { out := &Output{ SchemaVersion: SchemaVersion, Mod: make(map[string]*Package), + Install: install, + GoPackagePath: goPackagePath, } for _, pkg := range pkgs { diff --git a/tests/cli-args/default.nix b/tests/cli-args/default.nix new file mode 100644 index 0000000..2188edb --- /dev/null +++ b/tests/cli-args/default.nix @@ -0,0 +1,17 @@ +{ runCommand, buildGoApplication }: + +let + drv = buildGoApplication { + pname = "stringer"; + pwd = ./.; + }; +in +assert drv.version == "0.1.11"; +runCommand "cli-args-stringer-assert" { } '' + if ! test -f ${drv}/bin/stringer; then + echo "stringer command not found in env!" + exit 1 + fi + + ln -s ${drv} $out +'' diff --git a/tests/cli-args/script b/tests/cli-args/script new file mode 100755 index 0000000..36c0a46 --- /dev/null +++ b/tests/cli-args/script @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "${0%/*}" +exec $GOMOD2NIX generate "golang.org/x/tools/cmd/stringer@v0.1.11" diff --git a/tests/run.go b/tests/run.go index 7fabe93..a17c601 100644 --- a/tests/run.go +++ b/tests/run.go @@ -13,11 +13,15 @@ import ( "sync" ) -func runProcess(prefix string, command string, args ...string) error { +func runProcess(prefix string, env []string, command string, args ...string) error { fmt.Printf("%s: Executing %s %s\n", prefix, command, args) cmd := exec.Command(command, args...) + if env != nil { + cmd.Env = env + } + stdoutReader, err := cmd.StdoutPipe() if err != nil { return err @@ -61,18 +65,27 @@ func contains(haystack []string, needle string) bool { func runTest(rootDir string, testDir string) error { prefix := testDir - cmdPath := filepath.Join(rootDir, "..", "gomod2nix") testDir = filepath.Join(rootDir, testDir) + cmdPath := filepath.Join(rootDir, "..", "gomod2nix") - if _, err := os.Stat(filepath.Join(testDir, "go.mod")); err == nil { - err := runProcess(prefix, cmdPath, "--dir", testDir, "--outdir", testDir) + if _, err := os.Stat(filepath.Join(testDir, "script")); err == nil { + env := append(os.Environ(), "GOMOD2NIX="+cmdPath) + err := runProcess(prefix, env, filepath.Join(testDir, "script")) if err != nil { return err } + + } else { + if _, err := os.Stat(filepath.Join(testDir, "go.mod")); err == nil { + err := runProcess(prefix, nil, cmdPath, "--dir", testDir, "--outdir", testDir) + if err != nil { + return err + } + } } buildExpr := fmt.Sprintf("with (import { overlays = [ (import %s/../overlay.nix) ]; }); callPackage %s {}", rootDir, testDir) - err := runProcess(prefix, "nix-build", "--no-out-link", "--expr", buildExpr) + err := runProcess(prefix, nil, "nix-build", "--no-out-link", "--expr", buildExpr) if err != nil { return err }