Merge pull request #55 from adisbladis/import-sources

Add an import subcommand

cmd/root.go

@@ -0,0 +1,93 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	generate "github.com/tweag/gomod2nix/generate"
+	schema "github.com/tweag/gomod2nix/schema"
+)
+
+var (
+	flagDirectory string
+	flagOutDir    string
+	flagMaxJobs   int
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "gomod2nix",
+	Short: "Convert applications using Go modules -> Nix",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs)
+		if err != nil {
+			panic(err)
+		}
+	},
+}
+
+var generateCmd = &cobra.Command{
+	Use:   "generate",
+	Short: "Run gomod2nix.toml generator",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := generateInternal(flagDirectory, flagOutDir, flagMaxJobs)
+		if err != nil {
+			panic(err)
+		}
+	},
+}
+
+var importCmd = &cobra.Command{
+	Use:   "import",
+	Short: "Import Go sources into the Nix store",
+	Run: func(cmd *cobra.Command, args []string) {
+		err := generate.ImportPkgs(flagDirectory, flagMaxJobs)
+		if err != nil {
+			panic(err)
+		}
+	},
+}
+
+func init() {
+	rootCmd.PersistentFlags().StringVar(&flagDirectory, "dir", "", "Go project directory")
+	rootCmd.PersistentFlags().StringVar(&flagOutDir, "outdir", "", "Output directory (defaults to project directory)")
+	rootCmd.PersistentFlags().IntVar(&flagMaxJobs, "jobs", 10, "Max number of concurrent jobs")
+
+	rootCmd.AddCommand(generateCmd)
+	rootCmd.AddCommand(importCmd)
+}
+
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func generateInternal(directory string, outDir string, maxJobs int) error {
+	if outDir == "" {
+		outDir = directory
+	}
+
+	goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
+	outFile := goMod2NixPath
+	pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs)
+	if err != nil {
+		return fmt.Errorf("error generating pkgs: %v", err)
+	}
+
+	output, err := schema.Marshal(pkgs)
+	if err != nil {
+		return fmt.Errorf("error marshaling output: %v", err)
+	}
+
+	err = os.WriteFile(outFile, output, 0644)
+	if err != nil {
+		return fmt.Errorf("error writing file: %v", err)
+	}
+	log.Info(fmt.Sprintf("Wrote: %s", outFile))
+
+	return nil
+}

default.nix

@@ -1,4 +1,4 @@
-{ buildGoApplication, go, nix, lib, makeWrapper, nix-prefetch-git }:
+{ stdenv, buildGoApplication, go, nix, lib, makeWrapper, installShellFiles }:
 
 buildGoApplication {
   inherit go;
@@ -18,9 +18,21 @@ buildGoApplication {
 
   subPackages = [ "." ];
 
-  nativeBuildInputs = [ makeWrapper ];
+  nativeBuildInputs = [ makeWrapper installShellFiles ];
 
-  postInstall = ''
+  postInstall = lib.optionalString (stdenv.hostPlatform == stdenv.targetPlatform) ''
+    $out/bin/gomod2nix completion bash > gomod2nix.bash
+    $out/bin/gomod2nix completion fish > gomod2nix.fish
+    $out/bin/gomod2nix completion zsh > _gomod2nix
+    installShellCompletion gomod2nix.{bash,fish} _gomod2nix
+  '' + ''
     wrapProgram $out/bin/gomod2nix --prefix PATH : ${lib.makeBinPath [ go ]}
   '';
+
+  meta = {
+    description = "Convert applications using Go modules -> Nix";
+    homepage = "https://github.com/tweag/gomod2nix";
+    license = lib.licenses.mit;
+    maintainers = [ lib.maintainers.adisbladis ];
+  };
 }

generate/generate.go

@@ -5,8 +5,10 @@ import (
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"sort"
@@ -31,7 +33,11 @@ type goModDownload struct {
 	GoModSum string
 }
 
-func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) {
+func sourceFilter(name string, nodeType nar.NodeType) bool {
+	return strings.ToLower(filepath.Base(name)) != ".ds_store"
+}
+
+func common(directory string) ([]*goModDownload, map[string]string, error) {
 	goModPath := filepath.Join(directory, "go.mod")
 
 	log.WithFields(log.Fields{
@@ -41,13 +47,13 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc
 	// Read go.mod
 	data, err := ioutil.ReadFile(goModPath)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Parse go.mod
 	mod, err := modfile.Parse(goModPath, data, nil)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	// Map repos -> replacement repo
@@ -66,7 +72,7 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc
 		cmd.Dir = directory
 		stdout, err := cmd.Output()
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 
 		dec := json.NewDecoder(bytes.NewReader(stdout))
@@ -82,6 +88,65 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc
 		log.Info("Done downloading dependencies")
 	}
 
+	return modDownloads, replace, nil
+}
+
+func ImportPkgs(directory string, numWorkers int) error {
+	modDownloads, _, err := common(directory)
+	if err != nil {
+		return err
+	}
+
+	executor := lib.NewParallellExecutor(numWorkers)
+	for _, dl := range modDownloads {
+		dl := dl
+		executor.Add(func() error {
+			log.WithFields(log.Fields{
+				"goPackagePath": dl.Path,
+			}).Info("Importing sources")
+
+			pathName := filepath.Base(dl.Path) + "_" + dl.Version
+
+			cmd := exec.Command(
+				"nix-instantiate",
+				"--eval",
+				"--expr",
+				fmt.Sprintf(`
+builtins.filterSource (name: type: baseNameOf name != ".DS_Store") (
+  builtins.path {
+    path = "%s";
+    name = "%s";
+  }
+)
+`, dl.Dir, pathName),
+			)
+			cmd.Stderr = os.Stderr
+
+			err = cmd.Start()
+			if err != nil {
+				fmt.Println(cmd)
+				return err
+			}
+
+			err = cmd.Wait()
+			if err != nil {
+				fmt.Println(cmd)
+				return err
+			}
+
+			return nil
+		})
+	}
+
+	return executor.Wait()
+}
+
+func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) {
+	modDownloads, replace, err := common(directory)
+	if err != nil {
+		return nil, err
+	}
+
 	executor := lib.NewParallellExecutor(numWorkers)
 	var mux sync.Mutex
 
@@ -114,9 +179,7 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc
 			}).Info("Calculating NAR hash")
 
 			h := sha256.New()
-			err := nar.DumpPathFilter(h, dl.Dir, func(name string, nodeType nar.NodeType) bool {
+			err := nar.DumpPathFilter(h, dl.Dir, sourceFilter)
-				return strings.ToLower(filepath.Base(name)) != ".ds_store"
-			})
 			if err != nil {
 				return err
 			}

go.mod

@@ -10,6 +10,9 @@ require (
 )
 
 require (
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/spf13/cobra v1.4.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )

go.sum

@@ -13,6 +13,7 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger/v3 v3.2103.2/go.mod h1:RHo4/GmYcKKh5Lxu63wLEMHJ70Pac2JqZRYGhlyAo2M=
@@ -32,6 +33,7 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -48,6 +50,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -55,8 +58,12 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
+github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -128,6 +135,7 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

gomod2nix.toml

@@ -40,6 +40,9 @@ schema = 1
   [mod."github.com/cpuguy83/go-md2man"]
     version = "v1.0.10"
     hash = "sha256-XP8oKAx5LgQ0fO1rjO9tWnbXB431VOzXVaRDPUP900g="
+  [mod."github.com/cpuguy83/go-md2man/v2"]
+    version = "v2.0.1"
+    hash = "sha256-R+8iYit0mZjCxiGlp2C3Y5tGhySSNFU1v6wqsWKGBPM="
   [mod."github.com/davecgh/go-spew"]
     version = "v1.1.1"
     hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI="
@@ -130,6 +133,9 @@ schema = 1
   [mod."github.com/russross/blackfriday"]
     version = "v1.5.2"
     hash = "sha256-7MjJB7KZoNFTBo5Qzc3LL9QKrzWf9keqt5lh7tl360s="
+  [mod."github.com/russross/blackfriday/v2"]
+    version = "v2.1.0"
+    hash = "sha256-R+84l1si8az5yDqd5CYcFrTyNZ1eSYlpXKq6nFt4OTQ="
   [mod."github.com/sirupsen/logrus"]
     version = "v1.8.1"
     hash = "sha256-vUIDlLXYBD74+JqdoSx+W3J6r5cOk63heo0ElsHizoM="
@@ -143,14 +149,14 @@ schema = 1
     version = "v1.3.0"
     hash = "sha256-hbVF7F0YsgSybYEJa7W+Rz0As6OpgmpZOxB5JLFzAXc="
   [mod."github.com/spf13/cobra"]
-    version = "v0.0.5"
+    version = "v1.4.0"
-    hash = "sha256-YovRLduyDxoYoz1tNVgh9rnebjFIaeNeeBzWYrREnXw="
+    hash = "sha256-I6j9sD61Ztcc2W/WGeWo3ggYtnGTxNxZ2EFPdtO0UEY="
   [mod."github.com/spf13/jwalterweatherman"]
     version = "v1.0.0"
     hash = "sha256-KLftz+gaA5wSkvLqvQ7CuboB79kKEoTJvgTtrXatbiQ="
   [mod."github.com/spf13/pflag"]
-    version = "v1.0.3"
+    version = "v1.0.5"
-    hash = "sha256-Vu3dUmDZASu8/sAVAXaBXawtbZhsL9nMo43vANOdNMU="
+    hash = "sha256-w9LLYzxxP74WHT4ouBspH/iQZXjuAh2WQCHsuvyEjAw="
   [mod."github.com/spf13/viper"]
     version = "v1.3.2"
     hash = "sha256-8I/LyWr7cOKg+0NokEbewJ8WNz8hp0KLUW0WgNyGSaA="
@@ -218,8 +224,8 @@ schema = 1
     version = "v1.0.0-20190902080502-41f04d3bba15"
     hash = "sha256-Xl5gjoqU26M+Yxm6Xok5VHVpYT5hItwsN7d2Wj9L020="
   [mod."gopkg.in/yaml.v2"]
-    version = "v2.2.2"
+    version = "v2.4.0"
-    hash = "sha256-SjNLmGSxoYabOi/zdjymW3mSgaxaS39btBQ3/aUIkgc="
+    hash = "sha256-uVEGglIedjOIGZzHW4YwN1VoRSTK8o0eGZqzd+TNdd0="
   [mod."gopkg.in/yaml.v3"]
     version = "v3.0.0-20210107192922-496545a6307b"
     hash = "sha256-j8yDji+vqsitpRZirpb4w/Em8nstgf28wpwkcrOlxBk="

main.go

@@ -1,43 +1,7 @@
-package main // import "github.com/tweag/gomod2nix"
+package main
 
-import (
+import "github.com/tweag/gomod2nix/cmd"
-	"flag"
-	"fmt"
-	log "github.com/sirupsen/logrus"
-	generate "github.com/tweag/gomod2nix/generate"
-	schema "github.com/tweag/gomod2nix/schema"
-	"io/ioutil"
-	"path/filepath"
-)
 
 func main() {
-
+	cmd.Execute()
-	var directory = flag.String("dir", "./", "Go project directory")
-	var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
-	var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
-	flag.Parse()
-
-	outDir := *outDirFlag
-	if outDir == "" {
-		outDir = *directory
-	}
-
-	goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml")
-	outFile := goMod2NixPath
-	pkgs, err := generate.GeneratePkgs(*directory, goMod2NixPath, *maxJobs)
-	if err != nil {
-		panic(err)
-	}
-
-	output, err := schema.Marshal(pkgs)
-	if err != nil {
-		panic(err)
-	}
-
-	err = ioutil.WriteFile(outFile, output, 0644)
-	if err != nil {
-		panic(err)
-	}
-	log.Info(fmt.Sprintf("Wrote: %s", outFile))
-
 }