forked from mirrors/gomod2nix
fetch: Migrate to new fetcher method (using Go as the fetcher)
This commit is contained in:
adisbladis
parent
a5c4020da8
commit
f10ef7325c
4 changed files with 92 additions and 298 deletions
264
fetch/fetch.go
264
fetch/fetch.go
|
|
@ -1,22 +1,18 @@
|
|||
package fetch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/tweag/gomod2nix/formats/gomod2nix"
|
||||
"github.com/tweag/gomod2nix/types"
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/module"
|
||||
"golang.org/x/tools/go/vcs"
|
||||
)
|
||||
|
||||
type packageJob struct {
|
||||
|
|
@ -30,21 +26,15 @@ type packageResult struct {
|
|||
err error
|
||||
}
|
||||
|
||||
func worker(id int, cache map[string]*types.Package, jobs <-chan *packageJob, results chan<- *packageResult) {
|
||||
log.WithField("workerId", id).Info("Starting worker process")
|
||||
|
||||
for j := range jobs {
|
||||
log.WithFields(log.Fields{
|
||||
"workerId": id,
|
||||
"goPackagePath": j.goPackagePath,
|
||||
}).Info("Worker received job")
|
||||
|
||||
pkg, err := fetchPackage(cache, j.importPath, j.goPackagePath, j.sumVersion)
|
||||
results <- &packageResult{
|
||||
err: err,
|
||||
pkg: pkg,
|
||||
}
|
||||
}
|
||||
type goModDownload struct {
|
||||
Path string
|
||||
Version string
|
||||
Info string
|
||||
GoMod string
|
||||
Zip string
|
||||
Dir string
|
||||
Sum string
|
||||
GoModSum string
|
||||
}
|
||||
|
||||
func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, numWorkers int, keepGoing bool) ([]*types.Package, error) {
|
||||
|
|
@ -65,198 +55,80 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, num
|
|||
return nil, err
|
||||
}
|
||||
|
||||
caches := gomod2nix.LoadGomod2Nix(goMod2NixPath)
|
||||
|
||||
// Map repos -> replacement repo
|
||||
replace := make(map[string]string)
|
||||
for _, repl := range mod.Replace {
|
||||
replace[repl.New.Path] = repl.Old.Path
|
||||
}
|
||||
|
||||
var modDownloads []*goModDownload
|
||||
{
|
||||
log.WithFields(log.Fields{
|
||||
"sumPath": goSumPath,
|
||||
}).Info("Parsing go.sum")
|
||||
}).Info("Downloading dependencies")
|
||||
|
||||
sumVersions, err := parseGoSum(goSumPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
numJobs := len(sumVersions)
|
||||
if numJobs < numWorkers {
|
||||
numWorkers = numJobs
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"numWorkers": numWorkers,
|
||||
}).Info("Starting worker processes")
|
||||
jobs := make(chan *packageJob, numJobs)
|
||||
results := make(chan *packageResult, numJobs)
|
||||
for i := 0; i <= numWorkers; i++ {
|
||||
go worker(i, caches, jobs, results)
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"numJobs": numJobs,
|
||||
}).Info("Queuing jobs")
|
||||
for importPath, sumVersion := range sumVersions {
|
||||
// Check for replacement path (only original goPackagePath is recorded in go.sum)
|
||||
goPackagePath := importPath
|
||||
v, ok := replace[goPackagePath]
|
||||
if ok {
|
||||
goPackagePath = v
|
||||
}
|
||||
|
||||
jobs <- &packageJob{
|
||||
importPath: importPath,
|
||||
goPackagePath: goPackagePath,
|
||||
sumVersion: sumVersion,
|
||||
}
|
||||
}
|
||||
close(jobs)
|
||||
|
||||
var pkgs []*types.Package
|
||||
for i := 1; i <= numJobs; i++ {
|
||||
result := <-results
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"current": i,
|
||||
"total": numJobs,
|
||||
}).Info("Received finished job")
|
||||
|
||||
if result.err != nil {
|
||||
if keepGoing {
|
||||
fmt.Println(result.err)
|
||||
continue
|
||||
} else {
|
||||
return nil, result.err
|
||||
}
|
||||
}
|
||||
|
||||
pkgs = append(pkgs, result.pkg)
|
||||
}
|
||||
|
||||
sort.Slice(pkgs, func(i, j int) bool {
|
||||
return pkgs[i].GoPackagePath < pkgs[j].GoPackagePath
|
||||
})
|
||||
|
||||
return pkgs, nil
|
||||
}
|
||||
|
||||
func fetchPackage(cache map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) {
|
||||
repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z\-]*?[0-9]{14}-(.*?)$`)
|
||||
rev := strings.TrimSuffix(sumVersion, "+incompatible")
|
||||
if commitShaRev.MatchString(rev) {
|
||||
rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
|
||||
}
|
||||
|
||||
importPathPrefix, pathMajor, _ := module.SplitPathVersion(importPath)
|
||||
|
||||
// Relative path within the repo
|
||||
relPath := strings.TrimPrefix(importPathPrefix, repoRoot.Root+"/")
|
||||
if relPath == importPathPrefix {
|
||||
relPath = ""
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Checking previous invocation cache")
|
||||
{
|
||||
cached, ok := cache[goPackagePath]
|
||||
if ok {
|
||||
if cached.SumVersion == sumVersion {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Info("Returning cached entry")
|
||||
return cached, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if repoRoot.VCS.Name != "Git" {
|
||||
return nil, fmt.Errorf("Only git repositories are supported")
|
||||
}
|
||||
|
||||
type prefetchOutput struct {
|
||||
URL string `json:"url"`
|
||||
Rev string `json:"rev"`
|
||||
Sha256 string `json:"sha256"`
|
||||
Path string `json:"path"`
|
||||
// date string
|
||||
// fetchSubmodules bool
|
||||
// deepClone bool
|
||||
// leaveDotGit bool
|
||||
}
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
"rev": rev,
|
||||
}).Info("Cache miss, fetching")
|
||||
stdout, err := exec.Command(
|
||||
"nix-prefetch-git",
|
||||
"--quiet",
|
||||
"--fetch-submodules",
|
||||
"--url", repoRoot.Repo,
|
||||
"--rev", rev).Output()
|
||||
if err != nil {
|
||||
newRev := fmt.Sprintf("%s/%s", relPath, rev)
|
||||
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
"rev": newRev,
|
||||
}).Info("Fetching failed, retrying with different rev format")
|
||||
originalErr := err
|
||||
stdout, err = exec.Command(
|
||||
"nix-prefetch-git",
|
||||
"--quiet",
|
||||
"--fetch-submodules",
|
||||
"--url", repoRoot.Repo,
|
||||
"--rev", newRev).Output()
|
||||
if err != nil {
|
||||
log.WithFields(log.Fields{
|
||||
"goPackagePath": goPackagePath,
|
||||
}).Error("Fetching failed")
|
||||
return nil, originalErr
|
||||
}
|
||||
|
||||
rev = newRev
|
||||
}
|
||||
|
||||
var output *prefetchOutput
|
||||
err = json.Unmarshal(stdout, &output)
|
||||
"go", "mod", "download", "--json",
|
||||
).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
vendorPath := ""
|
||||
if importPath != goPackagePath {
|
||||
vendorPath = importPath
|
||||
dec := json.NewDecoder(bytes.NewReader(stdout))
|
||||
for {
|
||||
var dl *goModDownload
|
||||
err := dec.Decode(&dl)
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
modDownloads = append(modDownloads, dl)
|
||||
}
|
||||
|
||||
if relPath == "" && pathMajor != "" {
|
||||
p := filepath.Join(output.Path, pathMajor)
|
||||
_, err := os.Stat(p)
|
||||
if err == nil {
|
||||
fmt.Println(pathMajor)
|
||||
relPath = strings.TrimPrefix(pathMajor, "/")
|
||||
}
|
||||
log.WithFields(log.Fields{
|
||||
"sumPath": goSumPath,
|
||||
}).Info("Done downloading dependencies")
|
||||
}
|
||||
|
||||
return &types.Package{
|
||||
packages := []*types.Package{}
|
||||
for _, dl := range modDownloads {
|
||||
|
||||
goPackagePath, hasReplace := replace[dl.Path]
|
||||
if !hasReplace {
|
||||
goPackagePath = dl.Path
|
||||
}
|
||||
|
||||
var storePath string
|
||||
{
|
||||
stdout, err := exec.Command(
|
||||
"nix", "eval", "--impure", "--expr",
|
||||
fmt.Sprintf("builtins.path { name = \"%s_%s\"; path = \"%s\"; }", path.Base(goPackagePath), dl.Version, dl.Dir),
|
||||
).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
storePath = string(stdout)[1 : len(stdout)-2]
|
||||
}
|
||||
|
||||
stdout, err := exec.Command(
|
||||
"nix-store", "--query", "--hash", storePath,
|
||||
).Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash := strings.TrimSpace(string(stdout))
|
||||
|
||||
pkg := &types.Package{
|
||||
GoPackagePath: goPackagePath,
|
||||
URL: repoRoot.Repo,
|
||||
Rev: output.Rev,
|
||||
Sha256: output.Sha256,
|
||||
// This is used to skip fetching where the previous package path & versions are still the same
|
||||
// It's also used to construct the vendor directory in the Nix build
|
||||
SumVersion: sumVersion,
|
||||
RelPath: relPath,
|
||||
VendorPath: vendorPath,
|
||||
}, nil
|
||||
Version: dl.Version,
|
||||
Hash: hash,
|
||||
}
|
||||
if hasReplace {
|
||||
pkg.ReplacedPath = dl.Path
|
||||
}
|
||||
|
||||
packages = append(packages, pkg)
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
|
||||
}
|
||||
|
|
|
|||
34
fetch/sum.go
34
fetch/sum.go
|
|
@ -1,34 +0,0 @@
|
|||
package fetch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func parseGoSum(file string) (map[string]string, error) {
|
||||
|
||||
// Read go.mod
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
pkgs := make(map[string]string) // goPackagepath -> rev
|
||||
for lineno, line := range bytes.Split(data, []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
f := strings.Fields(string(line))
|
||||
if len(f) != 3 {
|
||||
return nil, fmt.Errorf("malformed go.sum:\n%s:%d: wrong number of fields %v", file, lineno, len(f))
|
||||
}
|
||||
|
||||
pkgs[f[0]] = strings.TrimSuffix(f[1], "/go.mod")
|
||||
}
|
||||
|
||||
return pkgs, nil
|
||||
|
||||
}
|
||||
|
|
@ -2,84 +2,43 @@ package gomod2nix
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"github.com/BurntSushi/toml"
|
||||
"github.com/tweag/gomod2nix/types"
|
||||
"io/ioutil"
|
||||
)
|
||||
|
||||
type fetchInfo struct {
|
||||
Type string `toml:"type"`
|
||||
URL string `toml:"url"`
|
||||
Rev string `toml:"rev"`
|
||||
Sha256 string `toml:"sha256"`
|
||||
}
|
||||
const schemaVersion = 1
|
||||
|
||||
type packageT struct {
|
||||
SumVersion string `toml:"sumVersion"`
|
||||
RelPath string `toml:"relPath,omitempty"`
|
||||
VendorPath string `toml:"vendorPath,omitempty"`
|
||||
Fetch *fetchInfo `toml:"fetch"`
|
||||
Version string `toml:"version"`
|
||||
Hash string `toml:"hash"`
|
||||
ReplacedPath string `toml:"replaced,omitempty"`
|
||||
}
|
||||
|
||||
type output struct {
|
||||
SchemaVersion int `toml:"schema"`
|
||||
Mod map[string]*packageT `toml:"mod"`
|
||||
}
|
||||
|
||||
func Marshal(pkgs []*types.Package) ([]byte, error) {
|
||||
result := make(map[string]*packageT)
|
||||
out := &output{
|
||||
SchemaVersion: schemaVersion,
|
||||
Mod: make(map[string]*packageT),
|
||||
}
|
||||
|
||||
for _, pkg := range pkgs {
|
||||
result[pkg.GoPackagePath] = &packageT{
|
||||
VendorPath: pkg.VendorPath,
|
||||
SumVersion: pkg.SumVersion,
|
||||
RelPath: pkg.RelPath,
|
||||
Fetch: &fetchInfo{
|
||||
Type: "git",
|
||||
URL: pkg.URL,
|
||||
Rev: pkg.Rev,
|
||||
Sha256: pkg.Sha256,
|
||||
},
|
||||
out.Mod[pkg.GoPackagePath] = &packageT{
|
||||
Version: pkg.Version,
|
||||
Hash: pkg.Hash,
|
||||
ReplacedPath: pkg.ReplacedPath,
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
e := toml.NewEncoder(&buf)
|
||||
err := e.Encode(result)
|
||||
err := e.Encode(out)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func LoadGomod2Nix(filePath string) map[string]*types.Package {
|
||||
ret := make(map[string]*types.Package)
|
||||
|
||||
if filePath == "" {
|
||||
return ret
|
||||
}
|
||||
|
||||
b, err := ioutil.ReadFile(filePath)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return ret
|
||||
}
|
||||
|
||||
result := make(map[string]*packageT)
|
||||
_, err = toml.Decode(string(b), &result)
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return ret
|
||||
}
|
||||
|
||||
for k, v := range result {
|
||||
ret[k] = &types.Package{
|
||||
GoPackagePath: k,
|
||||
URL: v.Fetch.URL,
|
||||
Rev: v.Fetch.Rev,
|
||||
Sha256: v.Fetch.Sha256,
|
||||
SumVersion: v.SumVersion,
|
||||
RelPath: v.RelPath,
|
||||
VendorPath: v.VendorPath,
|
||||
}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,7 @@ package types
|
|||
|
||||
type Package struct {
|
||||
GoPackagePath string
|
||||
URL string
|
||||
Rev string
|
||||
Sha256 string
|
||||
SumVersion string
|
||||
RelPath string
|
||||
VendorPath string
|
||||
ReplacedPath string
|
||||
Version string
|
||||
Hash string
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue