Add comprehensive logging

This commit is contained in:
adisbladis 2020-07-21 11:42:32 +02:00
parent 51402965c7
commit 25d7fc13aa
No known key found for this signature in database
GPG key ID: 110BFAD44C6249B7
4 changed files with 87 additions and 14 deletions

View file

@ -3,6 +3,7 @@ package fetch
import (
"encoding/json"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/tweag/gomod2nix/formats/buildgopackage"
"github.com/tweag/gomod2nix/formats/gomod2nix"
"github.com/tweag/gomod2nix/types"
@ -26,7 +27,14 @@ type packageResult struct {
}
func worker(id int, caches []map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) {
log.WithField("workerId", id).Info("Starting worker process")
for j := range jobs {
log.WithFields(log.Fields{
"workerId": id,
"goPackagePath": j.goPackagePath,
}).Info("Worker received job")
pkg, err := fetchPackage(caches, j.importPath, j.goPackagePath, j.rev)
results <- &packageResult{
err: err,
@ -42,6 +50,10 @@ func mkNewRev(goPackagePath string, repoRoot *vcs.RepoRoot, rev string) string {
func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, depsNixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
log.WithFields(log.Fields{
"modPath": goModPath,
}).Info("Parsing go.mod")
// Read go.mod
data, err := ioutil.ReadFile(goModPath)
if err != nil {
@ -54,9 +66,14 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
return nil, err
}
caches := []map[string]*types.Package{
gomod2nix.LoadGomod2Nix(goMod2NixPath),
buildgopackage.LoadDepsNix(depsNixPath),
caches := []map[string]*types.Package{}
goModCache := gomod2nix.LoadGomod2Nix(goMod2NixPath)
if len(goModCache) > 0 {
caches = append(caches, goModCache)
}
buildGoCache := buildgopackage.LoadDepsNix(depsNixPath)
if len(buildGoCache) > 0 {
caches = append(caches, buildGoCache)
}
// // Parse require
@ -71,6 +88,10 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
replace[repl.Old.Path] = repl.New.Path
}
log.WithFields(log.Fields{
"sumPath": goSumPath,
}).Info("Parsing go.sum")
revs, err := parseGoSum(goSumPath)
if err != nil {
return nil, err
@ -81,12 +102,18 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
numWorkers = numJobs
}
log.WithFields(log.Fields{
"numWorkers": numWorkers,
}).Info("Starting worker processes")
jobs := make(chan *packageJob, numJobs)
results := make(chan *packageResult, numJobs)
for i := 0; i <= numWorkers; i++ {
go worker(i, caches, jobs, results)
}
log.WithFields(log.Fields{
"numJobs": numJobs,
}).Info("Queuing jobs")
for goPackagePath, rev := range revs {
// Check for replacement path (only original goPackagePath is recorded in go.sum)
importPath := goPackagePath
@ -105,6 +132,12 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
var pkgs []*types.Package
for i := 1; i <= numJobs; i++ {
result := <-results
log.WithFields(log.Fields{
"current": i,
"total": numJobs,
}).Info("Received finished job")
if result.err != nil {
if keepGoing {
fmt.Println(result.err)
@ -131,12 +164,22 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
}
newRev := mkNewRev(goPackagePath, repoRoot, rev)
for _, cache := range caches {
cached, ok := cache[goPackagePath]
if ok {
for _, rev := range []string{rev, newRev} {
if cached.Rev == rev {
return cached, nil
if len(caches) > 0 {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Info("Checking previous invocation cache")
for _, cache := range caches {
cached, ok := cache[goPackagePath]
if ok {
for _, rev := range []string{rev, newRev} {
if cached.Rev == rev {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Info("Returning cached entry")
return cached, nil
}
}
}
}
@ -156,6 +199,11 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
// deepClone bool
// leaveDotGit bool
}
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
"rev": rev,
}).Info("Cache miss, fetching")
stdout, err := exec.Command(
"nix-prefetch-git",
"--quiet",
@ -163,6 +211,10 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
"--url", repoRoot.Repo,
"--rev", rev).Output()
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
"rev": newRev,
}).Info("Fetching failed, retrying with different rev format")
originalErr := err
stdout, err = exec.Command(
"nix-prefetch-git",
@ -171,6 +223,9 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
"--url", repoRoot.Repo,
"--rev", newRev).Output()
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, originalErr
}

1
go.mod
View file

@ -5,6 +5,7 @@ go 1.14
require (
github.com/BurntSushi/toml v0.3.1
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44
github.com/sirupsen/logrus v1.7.0
golang.org/x/mod v0.3.0
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
)

7
go.sum
View file

@ -9,6 +9,7 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkx
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -17,9 +18,13 @@ github.com/orivej/e v0.0.0-20180728214217-ac3492690fda/go.mod h1:eOxOguJBxQH6q/o
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44 h1:XDJpMiCKWt8CIT2LE1QrF4DdrvI1WciSNUrnYtNewPo=
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44/go.mod h1:4SkaXpoQ0tQ0OIkGqU8ByPLANmTTTU1iWPDz7YXatSA=
github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@ -31,6 +36,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

20
main.go
View file

@ -3,6 +3,7 @@ package main // import "github.com/tweag/gomod2nix"
import (
"flag"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/tweag/gomod2nix/fetch"
"github.com/tweag/gomod2nix/formats/buildgopackage"
"github.com/tweag/gomod2nix/formats/gomod2nix"
@ -15,10 +16,15 @@ func main() {
var keepGoing = flag.Bool("keep-going", false, "Whether to panic or not if a rev cannot be resolved (default \"false\")")
var directory = flag.String("dir", "./", "Go project directory")
var maxJobs = flag.Int("jobs", 10, "Number of max parallel jobs")
var outFile = flag.String("outfile", "gomod2nix.toml", "output file")
var outDirFlag = flag.String("outdir", "", "output directory (if different from project directory)")
var format = flag.String("format", "gomod2nix", "output format (gomod2nix, buildgopackage)")
flag.Parse()
outDir := *outDirFlag
if outDir == "" {
outDir = *directory
}
goSumPath := filepath.Join(*directory, "go.sum")
goModPath := filepath.Join(*directory, "go.mod")
@ -26,14 +32,18 @@ func main() {
goMod2NixPath := ""
depsNixPath := ""
outFile := ""
switch *format {
case "gomod2nix":
goMod2NixPath = filepath.Join(*directory, "gomod2nix.toml")
goMod2NixPath = filepath.Join(outDir, "gomod2nix.toml")
outFile = goMod2NixPath
case "buildgopackage":
depsNixPath = filepath.Join(*directory, "deps.nix")
depsNixPath = filepath.Join(outDir, "deps.nix")
outFile = depsNixPath
default:
panic(wrongFormatError)
}
log.Info(fmt.Sprintf("Using output format '%s'", *format))
pkgs, err := fetch.FetchPackages(goModPath, goSumPath, goMod2NixPath, depsNixPath, *maxJobs, *keepGoing)
if err != nil {
@ -56,10 +66,10 @@ func main() {
panic(wrongFormatError)
}
err = ioutil.WriteFile(*outFile, output, 0644)
err = ioutil.WriteFile(outFile, output, 0644)
if err != nil {
panic(err)
}
fmt.Println(fmt.Sprintf("Wrote: %s", *outFile))
log.Info(fmt.Sprintf("Wrote: %s", outFile))
}