Add package generation for non development packages

This makes it possible to generate packages that you do not have a
local checkout for, e.g. running:
`gomod2nix generate --outdir example/ golang.org/x/tools/cmd/stringer`

This will be useful for packaging dependencies that you are not
developing, but just simply packaging.
This commit is contained in:
adisbladis 2022-06-14 03:26:50 +08:00
parent 116b98475f
commit 1ffea526a0
12 changed files with 368 additions and 55 deletions

1
.gitignore vendored
View file

@ -1,3 +1,4 @@
/gomod2nix
result*
/tests/*/gomod2nix.toml
/tests/*/go.mod

View file

@ -151,8 +151,8 @@ let
};
buildGoApplication =
{ modules
, src
{ modules ? pwd + "/gomod2nix.toml"
, src ? pwd
, pwd ? null
, nativeBuildInputs ? [ ]
, allowGoReference ? false
@ -190,11 +190,19 @@ let
inherit go modulesStruct localReplaceCommands;
};
package = stdenv.mkDerivation (attrs // {
defaultPackage = modulesStruct.goPackagePath or "";
package = stdenv.mkDerivation (lib.optionalAttrs (defaultPackage != "")
{
pname = attrs.pname or baseNameOf defaultPackage;
version = lib.removePrefix "v" (modulesStruct.mod.${defaultPackage}).version;
} // attrs // {
nativeBuildInputs = [ removeReferencesTo go ] ++ nativeBuildInputs;
inherit (go) GOOS GOARCH;
inherit src;
GO_NO_VENDOR_CHECKS = "1";
GO111MODULE = "on";
@ -302,6 +310,12 @@ let
dir="$GOPATH/bin"
[ -e "$dir" ] && cp -r $dir $out
${lib.optionalString (lib.hasAttr "install" modulesStruct) ''
${lib.concatStringsSep "\n" (map (x: "go install ${x}") (modulesStruct.install or [ ]))}
mkdir -p $out/bin
cp -a $GOPATH/bin/* $out/bin/
''}
runHook postInstall
'';

View file

@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"io"
"os"
"path/filepath"
@ -11,39 +12,148 @@ import (
schema "github.com/tweag/gomod2nix/schema"
)
const directoryDefault = "./"
var (
flagDirectory string
flagOutDir string
flagMaxJobs int
maxJobs int
)
func generateFunc(cmd *cobra.Command, args []string) {
directory := flagDirectory
outDir := flagOutDir
// If we are dealing with a project packaged by passing packages on the command line
// we need to create a temporary project.
var tmpProj *generate.TempProject
if len(args) > 0 {
var err error
if directory != directoryDefault {
panic(fmt.Errorf("directory flag not supported together with import arguments"))
}
if outDir == "" {
pwd, err := os.Getwd()
if err != nil {
panic(err)
}
outDir = pwd
}
tmpProj, err = generate.NewTempProject(args)
if err != nil {
panic(err)
}
defer func() {
err := tmpProj.Remove()
if err != nil {
panic(err)
}
}()
directory = tmpProj.Dir
} else if outDir == "" {
// Default out to current working directory if we are developing some software in the current repo.
outDir = directory
}
// Write gomod2nix.toml
{
goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
outFile := goMod2NixPath
pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs)
if err != nil {
panic(fmt.Errorf("error generating pkgs: %v", err))
}
var goPackagePath string
var install []string
if tmpProj != nil {
install = tmpProj.Install
goPackagePath = tmpProj.GoPackagePath
}
output, err := schema.Marshal(pkgs, goPackagePath, install)
if err != nil {
panic(fmt.Errorf("error marshaling output: %v", err))
}
err = os.WriteFile(outFile, output, 0644)
if err != nil {
panic(fmt.Errorf("error writing file: %v", err))
}
log.Info(fmt.Sprintf("Wrote: %s", outFile))
}
// If we are dealing with a project packaged by passing packages on the command line, copy go.mod
if tmpProj != nil {
outMod := filepath.Join(outDir, "go.mod")
fin, err := os.Open(filepath.Join(tmpProj.Dir, "go.mod"))
if err != nil {
panic(fmt.Errorf("error opening go.mod: %v", err))
}
fout, err := os.Create(outMod)
if err != nil {
panic(fmt.Errorf("error creating go.mod: %v", err))
}
_, err = io.Copy(fout, fin)
if err != nil {
panic(fmt.Errorf("error writing go.mod: %v", err))
}
log.Info(fmt.Sprintf("Wrote: %s", outMod))
}
}
var rootCmd = &cobra.Command{
Use: "gomod2nix",
Short: "Convert applications using Go modules -> Nix",
Run: func(cmd *cobra.Command, args []string) {
err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs)
if err != nil {
panic(err)
}
},
Run: generateFunc,
}
var generateCmd = &cobra.Command{
Use: "generate",
Short: "Run gomod2nix.toml generator",
Run: func(cmd *cobra.Command, args []string) {
err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs)
if err != nil {
panic(err)
}
},
Run: generateFunc,
// func(cmd *cobra.Command, args []string) error {
// outDir := flagOutDir
// dir := flagDirectory
// if len(args) > 0 {
// tmpProj, err := generate.NewTempProject(args)
// if err != nil {
// panic(err)
// }
// defer tmpProj.Remove()
// dir = tmpProj.Dir
// outDir = "./tmp"
// }
// // dir := "/home/adisbladis/go/pkg/mod/github.com/kyleconroy/sqlc@v1.14.0"
// // fmt.Println(args)
// err := generateInternal(dir, outDir, flagMaxJobs)
// if err != nil {
// panic(err)
// }
// return nil
// },
}
var importCmd = &cobra.Command{
Use: "import",
Short: "Import Go sources into the Nix store",
Run: func(cmd *cobra.Command, args []string) {
err := generate.ImportPkgs(flagDirectory, flagMaxJobs)
err := generate.ImportPkgs(flagDirectory, maxJobs)
if err != nil {
panic(err)
}
@ -51,9 +161,9 @@ var importCmd = &cobra.Command{
}
func init() {
rootCmd.PersistentFlags().StringVar(&flagDirectory, "dir", "", "Go project directory")
rootCmd.PersistentFlags().StringVar(&flagDirectory, "dir", "./", "Go project directory")
rootCmd.PersistentFlags().StringVar(&flagOutDir, "outdir", "", "Output directory (defaults to project directory)")
rootCmd.PersistentFlags().IntVar(&flagMaxJobs, "jobs", 10, "Max number of concurrent jobs")
rootCmd.PersistentFlags().IntVar(&maxJobs, "jobs", 10, "Max number of concurrent jobs")
rootCmd.AddCommand(generateCmd)
rootCmd.AddCommand(importCmd)
@ -65,29 +175,3 @@ func Execute() {
os.Exit(1)
}
}
func generateInternal(directory string, outDir string, maxJobs int) error {
if outDir == "" {
outDir = directory
}
goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
outFile := goMod2NixPath
pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs)
if err != nil {
return fmt.Errorf("error generating pkgs: %v", err)
}
output, err := schema.Marshal(pkgs)
if err != nil {
return fmt.Errorf("error marshaling output: %v", err)
}
err = os.WriteFile(outFile, output, 0644)
if err != nil {
return fmt.Errorf("error writing file: %v", err)
}
log.Info(fmt.Sprintf("Wrote: %s", outFile))
return nil
}

View file

@ -1,4 +1,4 @@
package fetch
package generate
import (
"bytes"

166
generate/temp.go Normal file
View file

@ -0,0 +1,166 @@
package generate
import (
"fmt"
"go/ast"
"go/printer"
"go/token"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
log "github.com/sirupsen/logrus"
"golang.org/x/tools/go/vcs"
)
type TempProject struct {
Dir string
Install []string
GoPackagePath string
}
func NewTempProject(packages []string) (*TempProject, error) {
// Imports without version suffix
install := make([]string, len(packages))
for i, imp := range packages {
idx := strings.Index(imp, "@")
if idx == -1 {
idx = len(imp)
}
install[i] = imp[:idx]
}
var goPackagePath string
{
path := install[0]
log.WithFields(log.Fields{
"path": path,
}).Info("Finding repo root for import path")
repoRoot, err := vcs.RepoRootForImportPath(path, false)
if err != nil {
return nil, err
}
goPackagePath = repoRoot.Root
}
log.Info("Setting up temporary project")
dir, err := os.MkdirTemp("", "gomod2nix-proj")
if err != nil {
return nil, err
}
log.WithFields(log.Fields{
"dir": dir,
}).Info("Created temporary directory")
// Create tools.go
{
log.WithFields(log.Fields{
"dir": dir,
}).Info("Creating tools.go")
astFile := &ast.File{
Name: ast.NewIdent("main"),
Decls: []ast.Decl{
&ast.GenDecl{
Tok: token.IMPORT,
Specs: func() []ast.Spec {
specs := make([]ast.Spec, len(install))
i := 0
for _, imp := range install {
specs[i] = &ast.ImportSpec{
Name: ast.NewIdent("_"),
Path: &ast.BasicLit{
ValuePos: token.NoPos,
Kind: token.STRING,
Value: strconv.Quote(imp),
},
}
i++
}
return specs
}(),
},
},
}
f, err := os.Create(filepath.Join(dir, "tools.go"))
if err != nil {
return nil, fmt.Errorf("Error creating tools.go: %v", err)
}
defer f.Close()
fset := token.NewFileSet()
err = printer.Fprint(f, fset, astFile)
if err != nil {
return nil, fmt.Errorf("error writing tools.go: %v", err)
}
log.WithFields(log.Fields{
"dir": dir,
}).Info("Created tools.go")
}
// Set up go module
{
log.WithFields(log.Fields{
"dir": dir,
}).Info("Initializing go.mod")
cmd := exec.Command("go", "mod", "init", "gomod2nix/dummy/package")
cmd.Dir = dir
cmd.Stderr = os.Stderr
_, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("error creating go module: %v", err)
}
log.WithFields(log.Fields{
"dir": dir,
}).Info("Done initializing go.mod")
// For every dependency fetch it
for _, imp := range packages {
log.WithFields(log.Fields{
"dir": dir,
"dep": imp,
}).Info("Getting dependency")
cmd := exec.Command("go", "get", "-d", imp)
cmd.Dir = dir
cmd.Stderr = os.Stderr
_, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("error fetching '%s': %v", imp, err)
}
log.WithFields(log.Fields{
"dir": dir,
"dep": imp,
}).Info("Done getting dependency")
}
}
return &TempProject{
Dir: dir,
Install: install,
GoPackagePath: goPackagePath,
}, nil
}
func (t *TempProject) Remove() error {
return os.RemoveAll(t.Dir)
}

3
go.mod
View file

@ -6,12 +6,13 @@ require (
github.com/BurntSushi/toml v1.1.0
github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.4.0
golang.org/x/mod v0.5.1
golang.org/x/tools v0.0.0-20210106214847-113979e3529a
)
require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/spf13/cobra v1.4.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect

5
go.sum
View file

@ -15,6 +15,7 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M=
github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
@ -48,6 +49,7 @@ github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47 h1:K270nNzKXB
github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47/go.mod h1:eE1NFc/GHPnxAhTTuxIfB6JSvRQdMVQCMQqRrGRWWfQ=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@ -68,6 +70,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@ -119,6 +122,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -137,5 +141,6 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View file

@ -1,4 +1,4 @@
schema = 1
schema = 2
[mod]
[mod."cloud.google.com/go"]

View file

@ -6,7 +6,7 @@ import (
"os"
)
const SchemaVersion = 1
const SchemaVersion = 2
type Package struct {
GoPackagePath string `toml:"-"`
@ -18,12 +18,20 @@ type Package struct {
type Output struct {
SchemaVersion int `toml:"schema"`
Mod map[string]*Package `toml:"mod"`
// Packages with passed import paths trigger `go install` based on this list
Install []string `toml:"install,omitempty"`
// Packages with passed import paths has a "default package" which pname & version is inherit from
GoPackagePath string `toml:"goPackagePath,omitempty"`
}
func Marshal(pkgs []*Package) ([]byte, error) {
func Marshal(pkgs []*Package, goPackagePath string, install []string) ([]byte, error) {
out := &Output{
SchemaVersion: SchemaVersion,
Mod: make(map[string]*Package),
Install: install,
GoPackagePath: goPackagePath,
}
for _, pkg := range pkgs {

View file

@ -0,0 +1,17 @@
{ runCommand, buildGoApplication }:
let
drv = buildGoApplication {
pname = "stringer";
pwd = ./.;
};
in
assert drv.version == "0.1.11";
runCommand "cli-args-stringer-assert" { } ''
if ! test -f ${drv}/bin/stringer; then
echo "stringer command not found in env!"
exit 1
fi
ln -s ${drv} $out
''

4
tests/cli-args/script Executable file
View file

@ -0,0 +1,4 @@
#!/usr/bin/env bash
set -euo pipefail
cd "${0%/*}"
exec $GOMOD2NIX generate "golang.org/x/tools/cmd/stringer@v0.1.11"

View file

@ -13,11 +13,15 @@ import (
"sync"
)
func runProcess(prefix string, command string, args ...string) error {
func runProcess(prefix string, env []string, command string, args ...string) error {
fmt.Printf("%s: Executing %s %s\n", prefix, command, args)
cmd := exec.Command(command, args...)
if env != nil {
cmd.Env = env
}
stdoutReader, err := cmd.StdoutPipe()
if err != nil {
return err
@ -61,18 +65,27 @@ func contains(haystack []string, needle string) bool {
func runTest(rootDir string, testDir string) error {
prefix := testDir
cmdPath := filepath.Join(rootDir, "..", "gomod2nix")
testDir = filepath.Join(rootDir, testDir)
cmdPath := filepath.Join(rootDir, "..", "gomod2nix")
if _, err := os.Stat(filepath.Join(testDir, "go.mod")); err == nil {
err := runProcess(prefix, cmdPath, "--dir", testDir, "--outdir", testDir)
if _, err := os.Stat(filepath.Join(testDir, "script")); err == nil {
env := append(os.Environ(), "GOMOD2NIX="+cmdPath)
err := runProcess(prefix, env, filepath.Join(testDir, "script"))
if err != nil {
return err
}
} else {
if _, err := os.Stat(filepath.Join(testDir, "go.mod")); err == nil {
err := runProcess(prefix, nil, cmdPath, "--dir", testDir, "--outdir", testDir)
if err != nil {
return err
}
}
}
buildExpr := fmt.Sprintf("with (import <nixpkgs> { overlays = [ (import %s/../overlay.nix) ]; }); callPackage %s {}", rootDir, testDir)
err := runProcess(prefix, "nix-build", "--no-out-link", "--expr", buildExpr)
err := runProcess(prefix, nil, "nix-build", "--no-out-link", "--expr", buildExpr)
if err != nil {
return err
}