From 850c00dbbabf06da92beb6c0ca18e69d07775de1 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Tue, 10 May 2016 17:03:24 -0700 Subject: [PATCH] drone exec and drone agent now share code --- agent/agent.go | 1 + agent/updater.go | 18 ++- drone/agent/agent.go | 4 + drone/agent/exec.go | 244 +++-------------------------- model/job.go | 2 +- server/queue.go | 1 + store/datastore/ddl/mysql/4.sql | 9 ++ store/datastore/ddl/postgres/4.sql | 9 ++ store/datastore/ddl/sqlite3/4.sql | 9 ++ template/amber/build.amber | 14 +- 10 files changed, 79 insertions(+), 232 deletions(-) create mode 100644 store/datastore/ddl/mysql/4.sql create mode 100644 store/datastore/ddl/postgres/4.sql create mode 100644 store/datastore/ddl/sqlite3/4.sql diff --git a/agent/agent.go b/agent/agent.go index ff2d59ee..6a381393 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -62,6 +62,7 @@ func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error { a.Update(payload) return err } + a.Update(payload) err = a.exec(spec, payload, cancel) if err != nil { diff --git a/agent/updater.go b/agent/updater.go index d8d80541..d207876f 100644 --- a/agent/updater.go +++ b/agent/updater.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "sync" "time" "github.com/Sirupsen/logrus" @@ -41,10 +42,21 @@ func NewClientUpdater(client client.Client) UpdateFunc { } } -func NewClientLogger(w io.Writer) LoggerFunc { +func NewClientLogger(client client.Client, id int64, rc io.ReadCloser, wc io.WriteCloser) LoggerFunc { + var once sync.Once return func(line *build.Line) { + // annoying hack to only start streaming once the first line is written + once.Do(func() { + go func() { + err := client.Stream(id, rc) + if err != nil && err != io.ErrClosedPipe { + logrus.Errorf("Error streaming build logs. %s", err) + } + }() + }) + linejson, _ := json.Marshal(line) - w.Write(linejson) - w.Write([]byte{'\n'}) + wc.Write(linejson) + wc.Write([]byte{'\n'}) } } diff --git a/drone/agent/agent.go b/drone/agent/agent.go index 143ead3e..8f6eab31 100644 --- a/drone/agent/agent.go +++ b/drone/agent/agent.go @@ -141,6 +141,10 @@ func start(c *cli.Context) { } else { logrus.SetLevel(logrus.WarnLevel) } + logrus.Infof("Connecting to %s with token %s", + c.String("drone-server"), + c.String("drone-token"), + ) client := client.NewClientToken( c.String("drone-server"), diff --git a/drone/agent/exec.go b/drone/agent/exec.go index 3e03af75..1df73378 100644 --- a/drone/agent/exec.go +++ b/drone/agent/exec.go @@ -1,27 +1,15 @@ package agent import ( - "encoding/json" - "fmt" "io" - "regexp" - "strings" "time" "github.com/Sirupsen/logrus" - "github.com/dchest/uniuri" + "github.com/drone/drone/agent" + "github.com/drone/drone/build/docker" "github.com/drone/drone/client" - "github.com/drone/drone/engine/compiler" - "github.com/drone/drone/engine/compiler/builtin" - "github.com/drone/drone/engine/runner" - "github.com/drone/drone/engine/runner/docker" - "github.com/drone/drone/model" - "github.com/drone/drone/queue" - "github.com/drone/drone/version" - "github.com/drone/drone/yaml/expander" "github.com/samalba/dockerclient" - "golang.org/x/net/context" ) type config struct { @@ -48,233 +36,45 @@ func (r *pipeline) run() error { logrus.Infof("Starting build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - w.Job.Status = model.StatusRunning - w.Job.Started = time.Now().Unix() + cancel := make(chan bool, 1) + engine := docker.NewClient(r.docker) - prefix := fmt.Sprintf("drone_%s", uniuri.New()) + // streaming the logs + rc, wc := io.Pipe() + defer func() { + wc.Close() + rc.Close() + }() - envs := toEnv(w) - w.Yaml = expander.ExpandString(w.Yaml, envs) - - // inject the netrc file into the clone plugin if the repositroy is - // private and requires authentication. - var secrets []*model.Secret - if w.Verified { - secrets = append(secrets, w.Secrets...) + a := agent.Agent{ + Update: agent.NewClientUpdater(r.drone), + Logger: agent.NewClientLogger(r.drone, w.Job.ID, rc, wc), + Engine: engine, + Timeout: time.Minute * 15, + Platform: r.config.platform, + Namespace: r.config.namespace, + Escalate: r.config.privileged, + Pull: r.config.pull, } - if w.Repo.IsPrivate { - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_USERNAME", - Value: w.Netrc.Login, - Images: []string{"*"}, - Events: []string{"*"}, - }) - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_PASSWORD", - Value: w.Netrc.Password, - Images: []string{"*"}, - Events: []string{"*"}, - }) - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_MACHINE", - Value: w.Netrc.Machine, - Images: []string{"*"}, - Events: []string{"*"}, - }) - } - - var lastStatus string - if w.BuildLast != nil { - lastStatus = w.BuildLast.Status - } - - trans := []compiler.Transform{ - builtin.NewCloneOp(w.Repo.Kind, true), - builtin.NewSecretOp(w.Build.Event, secrets), - builtin.NewNormalizeOp(r.config.namespace), - builtin.NewWorkspaceOp("/drone", "/drone/src/github.com/"+w.Repo.FullName), - builtin.NewValidateOp( - w.Repo.IsTrusted, - r.config.whitelist, - ), - builtin.NewEnvOp(envs), - builtin.NewShellOp(builtin.Linux_adm64), - builtin.NewArgsOp(), - builtin.NewEscalateOp(r.config.privileged), - builtin.NewPodOp(prefix), - builtin.NewAliasOp(prefix), - builtin.NewPullOp(r.config.pull), - builtin.NewFilterOp( - lastStatus, - w.Build.Branch, - w.Build.Event, - w.Build.Deploy, - w.Job.Environment, - ), - } - - compile := compiler.New() - compile.Transforms(trans) - spec, err := compile.CompileString(w.Yaml) - if err != nil { - w.Job.Error = err.Error() - w.Job.ExitCode = 255 - w.Job.Finished = w.Job.Started - w.Job.Status = model.StatusError - pushRetry(r.drone, w) - return nil - } - - pushRetry(r.drone, w) - - conf := runner.Config{ - Engine: docker.New(r.docker), - } - - c := context.TODO() - c, timout := context.WithTimeout(c, time.Minute*time.Duration(w.Repo.Timeout)) - c, cancel := context.WithCancel(c) - defer cancel() - defer timout() - - run := conf.Runner(c, spec) - run.Run() - + // signal for canceling the build. wait := r.drone.Wait(w.Job.ID) defer wait.Cancel() go func() { if _, err := wait.Done(); err == nil { + cancel <- true logrus.Infof("Cancel build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - cancel() } }() - rc, wc := io.Pipe() - go func() { - // TODO(bradrydzewski) figure out how to resume upload on failure - err := r.drone.Stream(w.Job.ID, rc) - if err != nil && err != io.ErrClosedPipe { - logrus.Errorf("Error streaming build logs. %s", err) - } - }() + a.Run(w, cancel) - pipe := run.Pipe() - for { - line := pipe.Next() - if line == nil { - break - } - linejson, _ := json.Marshal(line) - wc.Write(linejson) - wc.Write([]byte{'\n'}) - } - - err = run.Wait() - - pipe.Close() wc.Close() rc.Close() - // catch the build result - if err != nil { - w.Job.ExitCode = 255 - } - if exitErr, ok := err.(*runner.ExitError); ok { - w.Job.ExitCode = exitErr.Code - } - - w.Job.Finished = time.Now().Unix() - - switch w.Job.ExitCode { - case 128, 130, 137: - w.Job.Status = model.StatusKilled - case 0: - w.Job.Status = model.StatusSuccess - default: - w.Job.Status = model.StatusFailure - } - - pushRetry(r.drone, w) - logrus.Infof("Finished build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) return nil } - -func pushRetry(client client.Client, w *queue.Work) { - for { - err := client.Push(w) - if err == nil { - return - } - logrus.Errorf("Error updating %s/%s#%d.%d. Retry in 30s. %s", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err) - logrus.Infof("Retry update in 30s") - time.Sleep(time.Second * 30) - } -} - -func toEnv(w *queue.Work) map[string]string { - envs := map[string]string{ - "CI": "drone", - "DRONE": "true", - "DRONE_ARCH": "linux_amd64", - "DRONE_REPO": w.Repo.FullName, - "DRONE_REPO_SCM": w.Repo.Kind, - "DRONE_REPO_OWNER": w.Repo.Owner, - "DRONE_REPO_NAME": w.Repo.Name, - "DRONE_REPO_LINK": w.Repo.Link, - "DRONE_REPO_AVATAR": w.Repo.Avatar, - "DRONE_REPO_BRANCH": w.Repo.Branch, - "DRONE_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate), - "DRONE_REPO_TRUSTED": fmt.Sprintf("%v", w.Repo.IsTrusted), - "DRONE_REMOTE_URL": w.Repo.Clone, - "DRONE_COMMIT_SHA": w.Build.Commit, - "DRONE_COMMIT_REF": w.Build.Ref, - "DRONE_COMMIT_BRANCH": w.Build.Branch, - "DRONE_COMMIT_LINK": w.Build.Link, - "DRONE_COMMIT_MESSAGE": w.Build.Message, - "DRONE_COMMIT_AUTHOR": w.Build.Author, - "DRONE_COMMIT_AUTHOR_EMAIL": w.Build.Email, - "DRONE_COMMIT_AUTHOR_AVATAR": w.Build.Avatar, - "DRONE_BUILD_NUMBER": fmt.Sprintf("%d", w.Build.Number), - "DRONE_BUILD_EVENT": w.Build.Event, - "DRONE_BUILD_STATUS": w.Build.Status, - "DRONE_BUILD_LINK": fmt.Sprintf("%s/%s/%d", w.System.Link, w.Repo.FullName, w.Build.Number), - "DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created), - "DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started), - "DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished), - "DRONE_YAML_VERIFIED": fmt.Sprintf("%v", w.Verified), - "DRONE_YAML_SIGNED": fmt.Sprintf("%v", w.Signed), - "DRONE_BRANCH": w.Build.Branch, - "DRONE_COMMIT": w.Build.Commit, - "DRONE_VERSION": version.Version, - } - - if w.Build.Event == model.EventTag { - envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/") - } - if w.Build.Event == model.EventPull { - envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref) - } - if w.Build.Event == model.EventDeploy { - envs["DRONE_DEPLOY_TO"] = w.Build.Deploy - } - - if w.BuildLast != nil { - envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status - envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number) - envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit - } - - // inject matrix values as environment variables - for key, val := range w.Job.Environment { - envs[key] = val - } - return envs -} - -var pullRegexp = regexp.MustCompile("\\d+") diff --git a/model/job.go b/model/job.go index 607d690f..b8d2bbd1 100644 --- a/model/job.go +++ b/model/job.go @@ -6,7 +6,7 @@ type Job struct { BuildID int64 `json:"-" meddler:"job_build_id"` NodeID int64 `json:"-" meddler:"job_node_id"` Number int `json:"number" meddler:"job_number"` - Error string `json:"error" meddler:"-"` + Error string `json:"error" meddler:"job_error"` Status string `json:"status" meddler:"job_status"` ExitCode int `json:"exit_code" meddler:"job_exit_code"` Enqueued int64 `json:"enqueued_at" meddler:"job_enqueued"` diff --git a/server/queue.go b/server/queue.go index 2230125a..e8afd8e6 100644 --- a/server/queue.go +++ b/server/queue.go @@ -91,6 +91,7 @@ func Update(c *gin.Context) { job.Finished = work.Job.Finished job.Status = work.Job.Status job.ExitCode = work.Job.ExitCode + job.Error = work.Job.Error if build.Status == model.StatusPending { build.Status = model.StatusRunning diff --git a/store/datastore/ddl/mysql/4.sql b/store/datastore/ddl/mysql/4.sql new file mode 100644 index 00000000..18d3330e --- /dev/null +++ b/store/datastore/ddl/mysql/4.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +ALTER TABLE jobs ADD COLUMN job_error VARCHAR(500); + +UPDATE jobs SET job_error = '' job_error = null; + +-- +migrate Down + +ALTER TABLE jobs DROP COLUMN job_error; diff --git a/store/datastore/ddl/postgres/4.sql b/store/datastore/ddl/postgres/4.sql new file mode 100644 index 00000000..b50f43cb --- /dev/null +++ b/store/datastore/ddl/postgres/4.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +ALTER TABLE jobs ADD COLUMN job_error VARCHAR(500); + +UPDATE jobs SET job_error = ''; + +-- +migrate Down + +ALTER TABLE jobs DROP COLUMN job_error; diff --git a/store/datastore/ddl/sqlite3/4.sql b/store/datastore/ddl/sqlite3/4.sql new file mode 100644 index 00000000..a57b6b10 --- /dev/null +++ b/store/datastore/ddl/sqlite3/4.sql @@ -0,0 +1,9 @@ +-- +migrate Up + +ALTER TABLE jobs ADD COLUMN job_error TEXT; + +UPDATE jobs SET job_error = ''; + +-- +migrate Down + +ALTER TABLE jobs DROP COLUMN job_error; diff --git a/template/amber/build.amber b/template/amber/build.amber index 55a117c8..32161baf 100644 --- a/template/amber/build.amber +++ b/template/amber/build.amber @@ -59,11 +59,11 @@ block content | pending assignment to a worker div[class="msg-running"] .hidden ? $job.Status != "running" - | started + | started span[data-livestamp=$job.Started] div[class="msg-finished"] .hidden ? $job.Finished == 0 - | finished + | finished span[data-livestamp=$job.Finished] div[class="msg-exited"] .hidden ? $job.Finished == 0 @@ -75,9 +75,12 @@ block content button.btn.btn-info.hidden#cancel cancel div.col-md-8 - pre#output - button.tail#tail - i.material-icons expand_more + if Job.Error != "" + div.alert.alert-danger #{Job.Error} + else + pre#output + button.tail#tail + i.material-icons expand_more block append scripts script @@ -88,4 +91,3 @@ block append scripts var status = #{json(Job.Status)}; var view = new JobViewModel(repo, build, job, status); -