Add comprehensive logging
This commit is contained in:
adisbladis
parent
51402965c7
commit
25d7fc13aa
4 changed files with 87 additions and 14 deletions
|
|
@ -3,6 +3,7 @@ package fetch
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tweag/gomod2nix/formats/buildgopackage"
|
||||
"github.com/tweag/gomod2nix/formats/gomod2nix"
|
||||
"github.com/tweag/gomod2nix/types"
|
||||
|
|
@ -26,7 +27,14 @@ type packageResult struct {
|
|||
}
|
||||
|
||||
func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) {
|
||||
log.WithField("workerId", id).Info("Starting worker process")
|
||||
|
||||
for j := range jobs {
|
||||
log.WithFields(log.Fields{
|
||||
"workerId": id,
|
||||
"goPackagePath": j.goPackagePath,
|
||||
}).Info("Worker received job")
|
||||
|
||||
pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.rev)
|
||||
results <- &packageResult{
|
||||
err: err,
|
||||
|
|
@ -42,6 +50,10 @@ func mkNewRev(goPackagePath string, repoRoot *vcs.RepoRoot, rev string) string {
|
|||
|
||||
func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, depsNixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"modPath": goModPath,
|
||||
}).Info("Parsing go.mod")
|
||||
|
||||
// Read go.mod
|
||||
data, err := ioutil.ReadFile(goModPath)
|
||||
if err != nil {
|
||||
|
|
@ -54,9 +66,14 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
|
|||
return nil, err
|
||||
}
|
||||
|
||||
caches := []map[string]*types.Package{
|
||||
gomod2nix.LoadGomod2Nix(goMod2NixPath),
|
||||
buildgopackage.LoadDepsNix(depsNixPath),
|
||||
caches := []map[string]*types.Package{}
|
||||
goModCache := gomod2nix.LoadGomod2Nix(goMod2NixPath)
|
||||
if len(goModCache) > 0 {
|
||||
caches = append(caches, goModCache)
|
||||
}
|
||||
buildGoCache := buildgopackage.LoadDepsNix(depsNixPath)
|
||||
if len(buildGoCache) > 0 {
|
||||
caches = append(caches, buildGoCache)
|
||||
}
|
||||
|
||||
// // Parse require
|
||||
|
|
@ -71,6 +88,10 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
|
|||
replace[repl.Old.Path] = repl.New.Path
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"sumPath": goSumPath,
|
||||
}).Info("Parsing go.sum")
|
||||
|
||||
revs, err := parseGoSum(goSumPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -81,12 +102,18 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
|
|||
numWorkers = numJobs
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"numWorkers": numWorkers,
|
||||
}).Info("Starting worker processes")
|
||||
jobs := make(chan *packageJob, numJobs)
|
||||
results := make(chan *packageResult, numJobs)
|
||||
for i := 0; i <= numWorkers; i++ {
|
||||
go worker(i, caches, jobs, results)
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"numJobs": numJobs,
|
||||
}).Info("Queuing jobs")
|
||||
for goPackagePath, rev := range revs {
|
||||
// Check for replacement path (only original goPackagePath is recorded in go.sum)
|
||||
importPath := goPackagePath
|
||||
|
|
@ -105,6 +132,12 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
|
|||
var pkgs []*types.Package
|
||||
for i := 1; i <= numJobs; i++ {
|
||||
result := <-results
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"current": i,
|
||||
"total": numJobs,
|
||||
}).Info("Received finished job")
|
||||
|
||||
if result.err != nil {
|
||||
if keepGoing {
|
||||
fmt.Println(result.err)
|
||||
|
|
@ -131,16 +164,26 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
|
|||
}
|
||||
|
||||
newRev := mkNewRev(goPackagePath, repoRoot, rev)
|
||||
if len(caches) > 0 {
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Checking previous invocation cache")
|
||||
|
||||
for _, cache := range caches {
|
||||
cached, ok := cache[goPackagePath]
|
||||
if ok {
|
||||
for _, rev := range []string{rev, newRev} {
|
||||
if cached.Rev == rev {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Returning cached entry")
|
||||
return cached, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if repoRoot.VCS.Name != "Git" {
|
||||
return nil, fmt.Errorf("Only git repositories are supported")
|
||||
|
|
@ -156,6 +199,11 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
|
|||
// deepClone bool
|
||||
// leaveDotGit bool
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
"rev": rev,
|
||||
}).Info("Cache miss, fetching")
|
||||
stdout, err := exec.Command(
|
||||
"nix-prefetch-git",
|
||||
"--quiet",
|
||||
|
|
@ -163,6 +211,10 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
|
|||
"--url", repoRoot.Repo,
|
||||
"--rev", rev).Output()
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
"rev": newRev,
|
||||
}).Info("Fetching failed, retrying with different rev format")
|
||||
originalErr := err
|
||||
stdout, err = exec.Command(
|
||||
"nix-prefetch-git",
|
||||
|
|
@ -171,6 +223,9 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
|
|||
"--url", repoRoot.Repo,
|
||||
"--rev", newRev).Output()
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Error("Fetching failed")
|
||||
return nil, originalErr
|
||||
}
|
||||
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -5,6 +5,7 @@ go 1.14
|
|||
require (
|
||||
github.com/BurntSushi/toml v0.3.1
|
||||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44
|
||||
github.com/sirupsen/logrus v1.7.0
|
||||
golang.org/x/mod v0.3.0
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
|
||||
)
|
||||
|
|
|
|||
7
go.sum
7
go.sum
|
|
@ -9,6 +9,7 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkx
|
|||
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
|
|
@ -17,9 +18,13 @@ github.com/orivej/e v0.0.0-20180728214217-ac3492690fda/go.mod h1:eOxOguJBxQH6q/o
|
|||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44 h1:XDJpMiCKWt8CIT2LE1QrF4DdrvI1WciSNUrnYtNewPo=
|
||||
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44/go.mod h1:4SkaXpoQ0tQ0OIkGqU8ByPLANmTTTU1iWPDz7YXatSA=
|
||||
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
|
|
@ -31,6 +36,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
|
|||
golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
|
|
|
|||
20
main.go
20
main.go
|
|
@ -3,6 +3,7 @@ package main // import "github.com/tweag/gomod2nix"
|
|||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tweag/gomod2nix/fetch"
|
||||
"github.com/tweag/gomod2nix/formats/buildgopackage"
|
||||
"github.com/tweag/gomod2nix/formats/gomod2nix"
|
||||
|
|
@ -15,10 +16,15 @@ func main() {
|
|||
var keepGoing = flag.Bool("keep-going", false, "Whether to panic or not if a rev cannot be resolved (default \"false\")")
|
||||
var directory = flag.String("dir", "./", "Go project directory")
|
||||
var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
|
||||
var outFile = flag.String("outfile", "gomod2nix.toml", "output file")
|
||||
var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
|
||||
var format = flag.String("format", "gomod2nix", "output format (gomod2nix, buildgopackage)")
|
||||
flag.Parse()
|
||||
|
||||
outDir := *outDirFlag
|
||||
if outDir == "" {
|
||||
outDir = *directory
|
||||
}
|
||||
|
||||
goSumPath := filepath.Join(*directory, "go.sum")
|
||||
goModPath := filepath.Join(*directory, "go.mod")
|
||||
|
||||
|
|
@ -26,14 +32,18 @@ func main() {
|
|||
|
||||
goMod2NixPath := ""
|
||||
depsNixPath := ""
|
||||
outFile := ""
|
||||
switch *format {
|
||||
case "gomod2nix":
|
||||
goMod2NixPath = filepath.Join(*directory, "gomod2nix.toml")
|
||||
goMod2NixPath = filepath.Join(outDir, "gomod2nix.toml")
|
||||
outFile = goMod2NixPath
|
||||
case "buildgopackage":
|
||||
depsNixPath = filepath.Join(*directory, "deps.nix")
|
||||
depsNixPath = filepath.Join(outDir, "deps.nix")
|
||||
outFile = depsNixPath
|
||||
default:
|
||||
panic(wrongFormatError)
|
||||
}
|
||||
log.Info(fmt.Sprintf("Using output format '%s'", *format))
|
||||
|
||||
pkgs, err := fetch.FetchPackages(goModPath, goSumPath, goMod2NixPath, depsNixPath, *maxJobs, *keepGoing)
|
||||
if err != nil {
|
||||
|
|
@ -56,10 +66,10 @@ func main() {
|
|||
panic(wrongFormatError)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(*outFile, output, 0644)
|
||||
err = ioutil.WriteFile(outFile, output, 0644)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(fmt.Sprintf("Wrote: %s", *outFile))
|
||||
log.Info(fmt.Sprintf("Wrote: %s", outFile))
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue