From 564e029c179aca4be2161e3f006bf31706cb5055 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Thu, 18 Jun 2015 17:36:52 -0700 Subject: [PATCH] refactor task to job --- cmd/drone-agent/updater.go | 8 +- cmd/drone-build/main.go | 4 +- cmd/drone-build/run.go | 8 +- cmd/drone-build/util.go | 4 +- pkg/runner/builtin/runner.go | 75 +++--- pkg/runner/builtin/updater.go | 22 +- pkg/runner/builtin/worker.go | 18 +- pkg/runner/runner.go | 12 +- pkg/server/commits.go | 32 ++- pkg/server/hooks.go | 8 +- pkg/server/queue.go | 19 +- pkg/server/ws.go | 8 +- pkg/store/builtin/build.go | 43 ++-- pkg/store/builtin/build_sql.go | 351 +++++++++++---------------- pkg/store/builtin/build_test.go | 131 +++++----- pkg/store/builtin/commit.go | 17 +- pkg/store/builtin/migrate/migrate.go | 25 +- pkg/store/builtin/store.go | 4 +- pkg/store/store.go | 14 +- pkg/types/build.go | 24 +- pkg/types/commit.go | 2 +- 21 files changed, 374 insertions(+), 455 deletions(-) diff --git a/cmd/drone-agent/updater.go b/cmd/drone-agent/updater.go index d763e9bc..a2da642a 100644 --- a/cmd/drone-agent/updater.go +++ b/cmd/drone-agent/updater.go @@ -22,13 +22,13 @@ func (u *updater) SetCommit(user *common.User, r *common.Repo, c *common.Commit) return sendBackoff("POST", path, c, nil) } -func (u *updater) SetBuild(r *common.Repo, c *common.Commit, b *common.Build) error { +func (u *updater) SetJob(r *common.Repo, c *common.Commit, j *common.Job) error { path := fmt.Sprintf("/api/queue/push/%s/%v", r.FullName, c.Sequence) - return sendBackoff("POST", path, b, nil) + return sendBackoff("POST", path, j, nil) } -func (u *updater) SetLogs(r *common.Repo, c *common.Commit, b *common.Build, rc io.ReadCloser) error { - path := fmt.Sprintf("/api/queue/push/%s/%v/%v", r.FullName, c.Sequence, b.Sequence) +func (u *updater) SetLogs(r *common.Repo, c *common.Commit, j *common.Job, rc io.ReadCloser) error { + path := fmt.Sprintf("/api/queue/push/%s/%v/%v", r.FullName, c.Sequence, j.Number) return sendBackoff("POST", path, rc, nil) } diff --git a/cmd/drone-build/main.go b/cmd/drone-build/main.go index 4aaf3863..8f7129a3 100644 --- a/cmd/drone-build/main.go +++ b/cmd/drone-build/main.go @@ -94,7 +94,7 @@ func main() { code = 255 } if code != 0 { - ctx.Build.ExitCode = code + ctx.Job.ExitCode = code break } } @@ -105,7 +105,7 @@ func main() { } client.Destroy() - os.Exit(ctx.Build.ExitCode) + os.Exit(ctx.Job.ExitCode) } func createClone(c *Context) error { diff --git a/cmd/drone-build/run.go b/cmd/drone-build/run.go index 28271c29..2d499229 100644 --- a/cmd/drone-build/run.go +++ b/cmd/drone-build/run.go @@ -14,7 +14,7 @@ type Context struct { Clone *common.Clone `json:"clone"` Repo *common.Repo `json:"repo"` Commit *common.Commit `json:"commit"` - Build *common.Build `json:"build"` + Job *common.Job `json:"job"` Keys *common.Keypair `json:"keys"` Netrc *common.Netrc `json:"netrc"` Yaml []byte `json:"yaml"` @@ -44,7 +44,7 @@ func setup(c *Context) error { } // inject the matrix parameters into the yaml - injected := inject.Inject(string(c.Yaml), c.Build.Environment) + injected := inject.Inject(string(c.Yaml), c.Job.Environment) c.Conf, err = parser.ParseSingle(injected, &opts) if err != nil { return err @@ -52,7 +52,7 @@ func setup(c *Context) error { // and append the matrix parameters as environment // variables for the build - for k, v := range c.Build.Environment { + for k, v := range c.Job.Environment { env := k + "=" + v c.Conf.Build.Environment = append(c.Conf.Build.Environment, env) } @@ -142,7 +142,7 @@ func runSteps(c *Context, steps map[string]*common.Step) (int, error) { if step.Condition != nil { if !step.Condition.MatchOwner(c.Repo.Owner) || !step.Condition.MatchBranch(c.Clone.Branch) || - !step.Condition.MatchMatrix(c.Build.Environment) { + !step.Condition.MatchMatrix(c.Job.Environment) { continue } } diff --git a/cmd/drone-build/util.go b/cmd/drone-build/util.go index 27349f00..8cb69293 100644 --- a/cmd/drone-build/util.go +++ b/cmd/drone-build/util.go @@ -70,7 +70,7 @@ func toCommand(c *Context, step *common.Step) []string { p := payload{ c.Repo, c.Commit, - c.Build, + c.Job, c.Clone, step.Config, } @@ -83,7 +83,7 @@ func toCommand(c *Context, step *common.Step) []string { type payload struct { Repo *common.Repo `json:"repo"` Commit *common.Commit `json:"commit"` - Build *common.Build `json:"build"` + Job *common.Job `json:"job"` Clone *common.Clone `json:"clone"` Config map[string]interface{} `json:"vargs"` diff --git a/pkg/runner/builtin/runner.go b/pkg/runner/builtin/runner.go index 6b83b53d..c71cea38 100644 --- a/pkg/runner/builtin/runner.go +++ b/pkg/runner/builtin/runner.go @@ -12,7 +12,7 @@ import ( "github.com/drone/drone/Godeps/_workspace/src/github.com/samalba/dockerclient" "github.com/drone/drone/pkg/docker" "github.com/drone/drone/pkg/queue" - common "github.com/drone/drone/pkg/types" + "github.com/drone/drone/pkg/types" log "github.com/drone/drone/Godeps/_workspace/src/github.com/Sirupsen/logrus" ) @@ -53,31 +53,29 @@ func (r *Runner) Run(w *queue.Work) error { // if any part of the commit fails and leaves // behind orphan sub-builds we need to cleanup // after ourselves. - if w.Commit.State == common.StateRunning { + if w.Commit.State == types.StateRunning { // if any tasks are running or pending // we should mark them as complete. for _, b := range w.Commit.Builds { - if b.State == common.StateRunning { - b.State = common.StateError + if b.Status == types.StateRunning { + b.Status = types.StateError b.Finished = time.Now().UTC().Unix() - b.Duration = b.Finished - b.Started b.ExitCode = 255 } - if b.State == common.StatePending { - b.State = common.StateError + if b.Status == types.StatePending { + b.Status = types.StateError b.Started = time.Now().UTC().Unix() b.Finished = time.Now().UTC().Unix() - b.Duration = 0 b.ExitCode = 255 } - r.SetBuild(w.Repo, w.Commit, b) + r.SetJob(w.Repo, w.Commit, b) } // must populate build start if w.Commit.Started == 0 { w.Commit.Started = time.Now().UTC().Unix() } // mark the build as complete (with error) - w.Commit.State = common.StateError + w.Commit.State = types.StateError w.Commit.Finished = time.Now().UTC().Unix() r.SetCommit(w.User, w.Repo, w.Commit) } @@ -85,7 +83,7 @@ func (r *Runner) Run(w *queue.Work) error { // marks the build as running w.Commit.Started = time.Now().UTC().Unix() - w.Commit.State = common.StateRunning + w.Commit.State = types.StateRunning err := r.SetCommit(w.User, w.Repo, w.Commit) if err != nil { return err @@ -100,13 +98,13 @@ func (r *Runner) Run(w *queue.Work) error { } // loop through and execute the build and - // clone steps for each build task. - for _, task := range w.Commit.Builds { + // clone steps for each build job. + for _, job := range w.Commit.Builds { // marks the task as running - task.State = common.StateRunning - task.Started = time.Now().UTC().Unix() - err = r.SetBuild(w.Repo, w.Commit, task) + job.Status = types.StateRunning + job.Started = time.Now().UTC().Unix() + err = r.SetJob(w.Repo, w.Commit, job) if err != nil { return err } @@ -117,7 +115,7 @@ func (r *Runner) Run(w *queue.Work) error { Keys: w.Keys, Netrc: w.Netrc, Yaml: w.Yaml, - Build: task, + Job: job, Env: w.Env, Plugins: w.Plugins, } @@ -128,20 +126,20 @@ func (r *Runner) Run(w *queue.Work) error { worker := newWorkerTimeout(client, w.Repo.Timeout) workers = append(workers, worker) - cname := cname(task) + cname := cname(job) pullrequest := (w.Commit.PullRequest != "") state, builderr := worker.Build(cname, in, pullrequest) switch { case builderr == ErrTimeout: - task.State = common.StateKilled + job.Status = types.StateKilled case builderr != nil: - task.State = common.StateError + job.Status = types.StateError case state != 0: - task.ExitCode = state - task.State = common.StateFailure + job.ExitCode = state + job.Status = types.StateFailure default: - task.State = common.StateSuccess + job.Status = types.StateSuccess } // send the logs to the datastore @@ -158,15 +156,14 @@ func (r *Runner) Run(w *queue.Work) error { defer rc.Close() docker.StdCopy(&buf, &buf, rc) } - err = r.SetLogs(w.Repo, w.Commit, task, ioutil.NopCloser(&buf)) + err = r.SetLogs(w.Repo, w.Commit, job, ioutil.NopCloser(&buf)) if err != nil { return err } // update the task in the datastore - task.Finished = time.Now().UTC().Unix() - task.Duration = task.Finished - task.Started - err = r.SetBuild(w.Repo, w.Commit, task) + job.Finished = time.Now().UTC().Unix() + err = r.SetJob(w.Repo, w.Commit, job) if err != nil { return err } @@ -174,10 +171,10 @@ func (r *Runner) Run(w *queue.Work) error { // update the build state if any of the sub-tasks // had a non-success status - w.Commit.State = common.StateSuccess - for _, build := range w.Commit.Builds { - if build.State != common.StateSuccess { - w.Commit.State = build.State + w.Commit.State = types.StateSuccess + for _, job := range w.Commit.Builds { + if job.Status != types.StateSuccess { + w.Commit.State = job.Status break } } @@ -188,14 +185,14 @@ func (r *Runner) Run(w *queue.Work) error { // loop through and execute the notifications and // the destroy all containers afterward. - for i, build := range w.Commit.Builds { + for i, job := range w.Commit.Builds { work := &work{ Repo: w.Repo, Commit: w.Commit, Keys: w.Keys, Netrc: w.Netrc, Yaml: w.Yaml, - Build: build, + Job: job, Env: w.Env, Plugins: w.Plugins, } @@ -210,21 +207,21 @@ func (r *Runner) Run(w *queue.Work) error { return nil } -func (r *Runner) Cancel(build *common.Build) error { +func (r *Runner) Cancel(job *types.Job) error { client, err := dockerclient.NewDockerClient(DockerHost, nil) if err != nil { return err } - return client.StopContainer(cname(build), 30) + return client.StopContainer(cname(job), 30) } -func (r *Runner) Logs(build *common.Build) (io.ReadCloser, error) { +func (r *Runner) Logs(job *types.Job) (io.ReadCloser, error) { client, err := dockerclient.NewDockerClient(DockerHost, nil) if err != nil { return nil, err } // make sure this container actually exists - info, err := client.InspectContainer(cname(build)) + info, err := client.InspectContainer(cname(job)) if err != nil { return nil, err } @@ -250,8 +247,8 @@ func (r *Runner) Logs(build *common.Build) (io.ReadCloser, error) { return client.ContainerLogs(info.Id, logOptsTail) } -func cname(build *common.Build) string { - return fmt.Sprintf("drone-%d", build.ID) +func cname(job *types.Job) string { + return fmt.Sprintf("drone-%d", job.ID) } func (r *Runner) Poll(q queue.Queue) { diff --git a/pkg/runner/builtin/updater.go b/pkg/runner/builtin/updater.go index 639cdd0a..a969c182 100644 --- a/pkg/runner/builtin/updater.go +++ b/pkg/runner/builtin/updater.go @@ -8,13 +8,13 @@ import ( "github.com/drone/drone/pkg/bus" "github.com/drone/drone/pkg/remote" "github.com/drone/drone/pkg/store" - common "github.com/drone/drone/pkg/types" + "github.com/drone/drone/pkg/types" ) type Updater interface { - SetCommit(*common.User, *common.Repo, *common.Commit) error - SetBuild(*common.Repo, *common.Commit, *common.Build) error - SetLogs(*common.Repo, *common.Commit, *common.Build, io.ReadCloser) error + SetCommit(*types.User, *types.Repo, *types.Commit) error + SetJob(*types.Repo, *types.Commit, *types.Job) error + SetLogs(*types.Repo, *types.Commit, *types.Job, io.ReadCloser) error } // NewUpdater returns an implementation of the Updater interface @@ -29,7 +29,7 @@ type updater struct { remote remote.Remote } -func (u *updater) SetCommit(user *common.User, r *common.Repo, c *common.Commit) error { +func (u *updater) SetCommit(user *types.User, r *types.Repo, c *types.Commit) error { err := u.store.SetCommit(c) if err != nil { return err @@ -45,7 +45,7 @@ func (u *updater) SetCommit(user *common.User, r *common.Repo, c *common.Commit) // build list. we should probably just rethink // the messaging instead of this hack. if c.Builds == nil || len(c.Builds) == 0 { - c.Builds, _ = u.store.BuildList(c) + c.Builds, _ = u.store.JobList(c) } msg, err := json.Marshal(c) @@ -61,8 +61,8 @@ func (u *updater) SetCommit(user *common.User, r *common.Repo, c *common.Commit) return nil } -func (u *updater) SetBuild(r *common.Repo, c *common.Commit, b *common.Build) error { - err := u.store.SetBuild(b) +func (u *updater) SetJob(r *types.Repo, c *types.Commit, j *types.Job) error { + err := u.store.SetJob(j) if err != nil { return err } @@ -72,7 +72,7 @@ func (u *updater) SetBuild(r *common.Repo, c *common.Commit, b *common.Build) er // build list. we should probably just rethink // the messaging instead of this hack. if c.Builds == nil || len(c.Builds) == 0 { - c.Builds, _ = u.store.BuildList(c) + c.Builds, _ = u.store.JobList(c) } msg, err := json.Marshal(c) @@ -88,7 +88,7 @@ func (u *updater) SetBuild(r *common.Repo, c *common.Commit, b *common.Build) er return nil } -func (u *updater) SetLogs(r *common.Repo, c *common.Commit, b *common.Build, rc io.ReadCloser) error { - path := fmt.Sprintf("/logs/%s/%v/%v", r.FullName, c.Sequence, b.Sequence) +func (u *updater) SetLogs(r *types.Repo, c *types.Commit, j *types.Job, rc io.ReadCloser) error { + path := fmt.Sprintf("/logs/%s/%v/%v", r.FullName, c.Sequence, j.Number) return u.store.SetBlobReader(path, rc) } diff --git a/pkg/runner/builtin/worker.go b/pkg/runner/builtin/worker.go index 5bb568f0..fe2ffb34 100644 --- a/pkg/runner/builtin/worker.go +++ b/pkg/runner/builtin/worker.go @@ -7,7 +7,7 @@ import ( "time" "github.com/drone/drone/Godeps/_workspace/src/github.com/samalba/dockerclient" - common "github.com/drone/drone/pkg/types" + "github.com/drone/drone/pkg/types" ) var ( @@ -52,14 +52,14 @@ var ( ) type work struct { - Repo *common.Repo `json:"repo"` - Commit *common.Commit `json:"commit"` - Build *common.Build `json:"build"` - Keys *common.Keypair `json:"keys"` - Netrc *common.Netrc `json:"netrc"` - Yaml []byte `json:"yaml"` - Env []string `json:"environment"` - Plugins []string `json:"plugins"` + Repo *types.Repo `json:"repo"` + Commit *types.Commit `json:"commit"` + Job *types.Job `json:"job"` + Keys *types.Keypair `json:"keys"` + Netrc *types.Netrc `json:"netrc"` + Yaml []byte `json:"yaml"` + Env []string `json:"environment"` + Plugins []string `json:"plugins"` } type worker struct { diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 9fd44d1d..dbfe5b4c 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -4,19 +4,19 @@ import ( "io" "github.com/drone/drone/pkg/queue" - common "github.com/drone/drone/pkg/types" + "github.com/drone/drone/pkg/types" ) type Runner interface { Run(work *queue.Work) error - Cancel(*common.Build) error - Logs(*common.Build) (io.ReadCloser, error) + Cancel(*types.Job) error + Logs(*types.Job) (io.ReadCloser, error) } // Updater defines a set of functions that are required for // the runner to sent Drone updates during a build. type Updater interface { - SetCommit(*common.User, *common.Repo, *common.Commit) error - SetBuild(*common.Repo, *common.Commit, *common.Build) error - SetLogs(*common.Repo, *common.Commit, *common.Build, io.ReadCloser) error + SetCommit(*types.User, *types.Repo, *types.Commit) error + SetJob(*types.Repo, *types.Commit, *types.Job) error + SetLogs(*types.Repo, *types.Commit, *types.Job, io.ReadCloser) error } diff --git a/pkg/server/commits.go b/pkg/server/commits.go index f687f21d..c0f85b28 100644 --- a/pkg/server/commits.go +++ b/pkg/server/commits.go @@ -31,7 +31,7 @@ func GetCommit(c *gin.Context) { if err != nil { c.Fail(404, err) } - commit.Builds, err = store.BuildList(commit) + commit.Builds, err = store.JobList(commit) if err != nil { c.Fail(404, err) } else { @@ -126,7 +126,7 @@ func RunBuild(c *gin.Context) { c.Fail(404, err) return } - commit.Builds, err = store.BuildList(commit) + commit.Builds, err = store.JobList(commit) if err != nil { c.Fail(404, err) return @@ -152,12 +152,11 @@ func RunBuild(c *gin.Context) { commit.State = common.StatePending commit.Started = 0 commit.Finished = 0 - for _, build := range commit.Builds { - build.State = common.StatePending - build.Started = 0 - build.Finished = 0 - build.Duration = 0 - build.ExitCode = 0 + for _, job := range commit.Builds { + job.Status = common.StatePending + job.Started = 0 + job.Finished = 0 + job.ExitCode = 0 } err = store.SetCommit(commit) @@ -217,7 +216,7 @@ func KillBuild(c *gin.Context) { c.Fail(404, err) return } - commit.Builds, err = store.BuildList(commit) + commit.Builds, err = store.JobList(commit) if err != nil { c.Fail(404, err) return @@ -245,14 +244,13 @@ func KillBuild(c *gin.Context) { if commit.Started == 0 { commit.Started = commit.Finished } - for _, build := range commit.Builds { - if build.State != common.StatePending && build.State != common.StateRunning { + for _, job := range commit.Builds { + if job.Status != common.StatePending && job.Status != common.StateRunning { continue } - build.State = common.StateKilled - build.Started = commit.Started - build.Finished = commit.Finished - build.Duration = commit.Finished - commit.Started + job.Status = common.StateKilled + job.Started = commit.Started + job.Finished = commit.Finished } err = store.SetCommit(commit) if err != nil { @@ -260,8 +258,8 @@ func KillBuild(c *gin.Context) { return } - for _, build := range commit.Builds { - runner.Cancel(build) + for _, job := range commit.Builds { + runner.Cancel(job) } // // get the agent from the repository so we can // // notify the agent to kill the build. diff --git a/pkg/server/hooks.go b/pkg/server/hooks.go index 89e241f0..f3283c1d 100644 --- a/pkg/server/hooks.go +++ b/pkg/server/hooks.go @@ -110,10 +110,10 @@ func PostHook(c *gin.Context) { axes = append(axes, matrix.Axis{}) } for num, axis := range axes { - commit.Builds = append(commit.Builds, &common.Build{ - CommitID: commit.ID, - Sequence: num + 1, - State: common.StatePending, + commit.Builds = append(commit.Builds, &common.Job{ + BuildID: commit.ID, + Number: num + 1, + Status: common.StatePending, Environment: axis, }) } diff --git a/pkg/server/queue.go b/pkg/server/queue.go index baeb3f05..3ec96e34 100644 --- a/pkg/server/queue.go +++ b/pkg/server/queue.go @@ -89,7 +89,7 @@ func PushBuild(c *gin.Context) { repo := ToRepo(c) cnum, _ := strconv.Atoi(c.Params.ByName("commit")) - in := &common.Build{} + in := &common.Job{} if !c.BindWith(in, binding.JSON) { return } @@ -99,20 +99,19 @@ func PushBuild(c *gin.Context) { c.Fail(404, err) return } - build, err := store.BuildSeq(commit, in.Sequence) + job, err := store.JobNumber(commit, in.Number) if err != nil { c.Fail(404, err) return } - build.Duration = in.Duration - build.Started = in.Started - build.Finished = in.Finished - build.ExitCode = in.ExitCode - build.State = in.State + job.Started = in.Started + job.Finished = in.Finished + job.ExitCode = in.ExitCode + job.Status = in.Status updater := ToUpdater(c) - err = updater.SetBuild(repo, commit, build) + err = updater.SetJob(repo, commit, job) if err != nil { c.Fail(500, err) return @@ -132,13 +131,13 @@ func PushLogs(c *gin.Context) { c.Fail(404, err) return } - build, err := store.BuildSeq(commit, bnum) + job, err := store.JobNumber(commit, bnum) if err != nil { c.Fail(404, err) return } updater := ToUpdater(c) - err = updater.SetLogs(repo, commit, build, c.Request.Body) + err = updater.SetLogs(repo, commit, job, c.Request.Body) if err != nil { c.Fail(500, err) return diff --git a/pkg/server/ws.go b/pkg/server/ws.go index 55adde1c..7abc6f9f 100644 --- a/pkg/server/ws.go +++ b/pkg/server/ws.go @@ -56,7 +56,7 @@ func GetStream(c *gin.Context) { repo := ToRepo(c) runner := ToRunner(c) commitseq, _ := strconv.Atoi(c.Params.ByName("build")) - buildseq, _ := strconv.Atoi(c.Params.ByName("number")) + jobnum, _ := strconv.Atoi(c.Params.ByName("number")) c.Writer.Header().Set("Content-Type", "text/event-stream") @@ -65,7 +65,7 @@ func GetStream(c *gin.Context) { c.Fail(404, err) return } - build, err := store.BuildSeq(commit, buildseq) + job, err := store.JobNumber(commit, jobnum) if err != nil { c.Fail(404, err) return @@ -82,7 +82,7 @@ func GetStream(c *gin.Context) { c.Fail(500, err) return } - url := fmt.Sprintf("http://%s/stream/%d?token=%s", addr, build.ID, conf.Agents.Secret) + url := fmt.Sprintf("http://%s/stream/%d?token=%s", addr, job.ID, conf.Agents.Secret) resp, err := http.Get(url) if err != nil { c.Fail(500, err) @@ -97,7 +97,7 @@ func GetStream(c *gin.Context) { } else { // else if the commit is not being executed // by the build agent we can use the local runner - rc, err = runner.Logs(build) + rc, err = runner.Logs(job) if err != nil { c.Fail(404, err) return diff --git a/pkg/store/builtin/build.go b/pkg/store/builtin/build.go index 0a1d8211..e4442b49 100644 --- a/pkg/store/builtin/build.go +++ b/pkg/store/builtin/build.go @@ -2,46 +2,39 @@ package builtin import ( "database/sql" - "time" "github.com/drone/drone/pkg/types" ) -type Buildstore struct { +type Jobstore struct { *sql.DB } -func NewBuildstore(db *sql.DB) *Buildstore { - return &Buildstore{db} +func NewJobstore(db *sql.DB) *Jobstore { + return &Jobstore{db} } -// Build returns a build by ID. -func (db *Buildstore) Build(id int64) (*types.Build, error) { - return getBuild(db, rebind(stmtBuildSelect), id) +// Job returns a Job by ID. +func (db *Jobstore) Job(id int64) (*types.Job, error) { + return getJob(db, rebind(stmtJobSelect), id) } -// BuildSeq returns a build by sequence number. -func (db *Buildstore) BuildSeq(commit *types.Commit, seq int) (*types.Build, error) { - return getBuild(db, rebind(stmtBuildSelectBuildSeq), commit.ID, seq) +// JobNumber returns a job by sequence number. +func (db *Jobstore) JobNumber(commit *types.Commit, seq int) (*types.Job, error) { + return getJob(db, rebind(stmtJobSelectBuildNumber), commit.ID, seq) } -// BuildList returns a list of all commit builds -func (db *Buildstore) BuildList(commit *types.Commit) ([]*types.Build, error) { - return getBuilds(db, rebind(stmtBuildSelectBuildCommitId), commit.ID) +// JobList returns a list of all build jobs +func (db *Jobstore) JobList(commit *types.Commit) ([]*types.Job, error) { + return getJobs(db, rebind(stmtJobSelectJobBuildId), commit.ID) } -// SetBuild updates an existing build. -func (db *Buildstore) SetBuild(build *types.Build) error { - build.Updated = time.Now().UTC().Unix() - return updateBuild(db, rebind(stmtBuildUpdate), build) +// SetJob updates an existing build job. +func (db *Jobstore) SetJob(job *types.Job) error { + return updateJob(db, rebind(stmtJobUpdate), job) } -// AddBuild inserts a build. -func (db *Buildstore) AddBuild(build *types.Build) error { - build.Created = time.Now().UTC().Unix() - build.Updated = time.Now().UTC().Unix() - return createBuild(db, rebind(stmtBuildInsert), build) +// AddJob inserts a build job. +func (db *Jobstore) AddJob(job *types.Job) error { + return createJob(db, rebind(stmtJobInsert), job) } - -// Build table name in database. -const buildTable = "builds" diff --git a/pkg/store/builtin/build_sql.go b/pkg/store/builtin/build_sql.go index 1d5d02af..317a73ac 100644 --- a/pkg/store/builtin/build_sql.go +++ b/pkg/store/builtin/build_sql.go @@ -13,47 +13,41 @@ import ( var _ = json.Marshal // generic database interface, matching both *sql.Db and *sql.Tx -type buildDB interface { +type jobDB interface { Exec(query string, args ...interface{}) (sql.Result, error) Query(query string, args ...interface{}) (*sql.Rows, error) QueryRow(query string, args ...interface{}) *sql.Row } -func getBuild(db buildDB, query string, args ...interface{}) (*Build, error) { +func getJob(db jobDB, query string, args ...interface{}) (*Job, error) { row := db.QueryRow(query, args...) - return scanBuild(row) + return scanJob(row) } -func getBuilds(db buildDB, query string, args ...interface{}) ([]*Build, error) { +func getJobs(db jobDB, query string, args ...interface{}) ([]*Job, error) { rows, err := db.Query(query, args...) if err != nil { return nil, err } defer rows.Close() - return scanBuilds(rows) + return scanJobs(rows) } -func createBuild(db buildDB, query string, v *Build) error { +func createJob(db jobDB, query string, v *Job) error { var v0 int64 - var v1 string - var v2 int + var v1 int + var v2 string var v3 int var v4 int64 var v5 int64 - var v6 int64 - var v7 int64 - var v8 int64 - var v9 []byte - v0 = v.CommitID - v1 = v.State - v2 = v.ExitCode - v3 = v.Sequence - v4 = v.Duration - v5 = v.Started - v6 = v.Finished - v7 = v.Created - v8 = v.Updated - v9, _ = json.Marshal(v.Environment) + var v6 []byte + v0 = v.BuildID + v1 = v.Number + v2 = v.Status + v3 = v.ExitCode + v4 = v.Started + v5 = v.Finished + v6, _ = json.Marshal(v.Environment) res, err := db.Exec(query, &v0, @@ -63,9 +57,6 @@ func createBuild(db buildDB, query string, v *Build) error { &v4, &v5, &v6, - &v7, - &v8, - &v9, ) if err != nil { return err @@ -75,29 +66,23 @@ func createBuild(db buildDB, query string, v *Build) error { return err } -func updateBuild(db buildDB, query string, v *Build) error { +func updateJob(db jobDB, query string, v *Job) error { var v0 int64 var v1 int64 - var v2 string - var v3 int + var v2 int + var v3 string var v4 int var v5 int64 var v6 int64 - var v7 int64 - var v8 int64 - var v9 int64 - var v10 []byte + var v7 []byte v0 = v.ID - v1 = v.CommitID - v2 = v.State - v3 = v.ExitCode - v4 = v.Sequence - v5 = v.Duration - v6 = v.Started - v7 = v.Finished - v8 = v.Created - v9 = v.Updated - v10, _ = json.Marshal(v.Environment) + v1 = v.BuildID + v2 = v.Number + v3 = v.Status + v4 = v.ExitCode + v5 = v.Started + v6 = v.Finished + v7, _ = json.Marshal(v.Environment) _, err := db.Exec(query, &v1, @@ -107,26 +92,20 @@ func updateBuild(db buildDB, query string, v *Build) error { &v5, &v6, &v7, - &v8, - &v9, - &v10, &v0, ) return err } -func scanBuild(row *sql.Row) (*Build, error) { +func scanJob(row *sql.Row) (*Job, error) { var v0 int64 var v1 int64 - var v2 string - var v3 int + var v2 int + var v3 string var v4 int var v5 int64 var v6 int64 - var v7 int64 - var v8 int64 - var v9 int64 - var v10 []byte + var v7 []byte err := row.Scan( &v0, @@ -137,45 +116,36 @@ func scanBuild(row *sql.Row) (*Build, error) { &v5, &v6, &v7, - &v8, - &v9, - &v10, ) if err != nil { return nil, err } - v := &Build{} + v := &Job{} v.ID = v0 - v.CommitID = v1 - v.State = v2 - v.ExitCode = v3 - v.Sequence = v4 - v.Duration = v5 - v.Started = v6 - v.Finished = v7 - v.Created = v8 - v.Updated = v9 - json.Unmarshal(v10, &v.Environment) + v.BuildID = v1 + v.Number = v2 + v.Status = v3 + v.ExitCode = v4 + v.Started = v5 + v.Finished = v6 + json.Unmarshal(v7, &v.Environment) return v, nil } -func scanBuilds(rows *sql.Rows) ([]*Build, error) { +func scanJobs(rows *sql.Rows) ([]*Job, error) { var err error - var vv []*Build + var vv []*Job for rows.Next() { var v0 int64 var v1 int64 - var v2 string - var v3 int + var v2 int + var v3 string var v4 int var v5 int64 var v6 int64 - var v7 int64 - var v8 int64 - var v9 int64 - var v10 []byte + var v7 []byte err = rows.Scan( &v0, &v1, @@ -185,171 +155,146 @@ func scanBuilds(rows *sql.Rows) ([]*Build, error) { &v5, &v6, &v7, - &v8, - &v9, - &v10, ) if err != nil { return vv, err } - v := &Build{} + v := &Job{} v.ID = v0 - v.CommitID = v1 - v.State = v2 - v.ExitCode = v3 - v.Sequence = v4 - v.Duration = v5 - v.Started = v6 - v.Finished = v7 - v.Created = v8 - v.Updated = v9 - json.Unmarshal(v10, &v.Environment) + v.BuildID = v1 + v.Number = v2 + v.Status = v3 + v.ExitCode = v4 + v.Started = v5 + v.Finished = v6 + json.Unmarshal(v7, &v.Environment) vv = append(vv, v) } return vv, rows.Err() } -const stmtBuildSelectList = ` +const stmtJobSelectList = ` SELECT - build_id -,build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -FROM builds + job_id +,job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +FROM jobs ` -const stmtBuildSelectRange = ` +const stmtJobSelectRange = ` SELECT - build_id -,build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -FROM builds + job_id +,job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +FROM jobs LIMIT ? OFFSET ? ` -const stmtBuildSelect = ` +const stmtJobSelect = ` SELECT - build_id -,build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -FROM builds -WHERE build_id = ? + job_id +,job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +FROM jobs +WHERE job_id = ? ` -const stmtBuildSelectBuildCommitId = ` +const stmtJobSelectJobBuildId = ` SELECT - build_id -,build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -FROM builds -WHERE build_commit_id = ? + job_id +,job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +FROM jobs +WHERE job_build_id = ? ` -const stmtBuildSelectBuildSeq = ` +const stmtJobSelectBuildNumber = ` SELECT - build_id -,build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -FROM builds -WHERE build_commit_id = ? -AND build_sequence = ? + job_id +,job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +FROM jobs +WHERE job_build_id = ? +AND job_number = ? ` -const stmtBuildInsert = ` -INSERT INTO builds ( - build_commit_id -,build_state -,build_exit_code -,build_sequence -,build_duration -,build_started -,build_finished -,build_created -,build_updated -,build_environment -) VALUES (?,?,?,?,?,?,?,?,?,?); +const stmtJobSelectCount = ` +SELECT count(1) +FROM jobs ` -const stmtBuildUpdate = ` -UPDATE builds SET - build_commit_id = ? -,build_state = ? -,build_exit_code = ? -,build_sequence = ? -,build_duration = ? -,build_started = ? -,build_finished = ? -,build_created = ? -,build_updated = ? -,build_environment = ? -WHERE build_id = ? +const stmtJobInsert = ` +INSERT INTO jobs ( + job_build_id +,job_number +,job_status +,job_exit_code +,job_started +,job_finished +,job_environment +) VALUES (?,?,?,?,?,?,?); ` -const stmtBuildDelete = ` -DELETE FROM builds -WHERE build_id = ? +const stmtJobUpdate = ` +UPDATE jobs SET + job_build_id = ? +,job_number = ? +,job_status = ? +,job_exit_code = ? +,job_started = ? +,job_finished = ? +,job_environment = ? +WHERE job_id = ? ` -const stmtBuildTable = ` -CREATE TABLE IF NOT EXISTS builds ( - build_id INTEGER PRIMARY KEY AUTOINCREMENT -,build_commit_id INTEGER -,build_state VARCHAR -,build_exit_code INTEGER -,build_sequence INTEGER -,build_duration INTEGER -,build_started INTEGER -,build_finished INTEGER -,build_created INTEGER -,build_updated INTEGER -,build_environment VARCHAR(2048) +const stmtJobDelete = ` +DELETE FROM jobs +WHERE job_id = ? +` + +const stmtJobTable = ` +CREATE TABLE IF NOT EXISTS jobs ( + job_id INTEGER PRIMARY KEY AUTOINCREMENT +,job_build_id INTEGER +,job_number INTEGER +,job_status VARCHAR +,job_exit_code INTEGER +,job_started INTEGER +,job_finished INTEGER +,job_environmentVARCHAR(2048) ); ` -const stmtBuildBuildCommitIdIndex = ` -CREATE INDEX IF NOT EXISTS ix_build_commit_id ON builds (build_commit_id); +const stmtJobJobBuildIdIndex = ` +CREATE INDEX IF NOT EXISTS ix_job_build_id ON jobs (job_build_id); ` -const stmtBuildBuildSeqIndex = ` -CREATE UNIQUE INDEX IF NOT EXISTS ux_build_seq ON builds (build_commit_id,build_sequence); +const stmtJobBuildNumberIndex = ` +CREATE UNIQUE INDEX IF NOT EXISTS ux_build_number ON jobs (job_build_id,job_number); ` diff --git a/pkg/store/builtin/build_test.go b/pkg/store/builtin/build_test.go index 8db23f4d..e3b45842 100644 --- a/pkg/store/builtin/build_test.go +++ b/pkg/store/builtin/build_test.go @@ -9,118 +9,117 @@ import ( func TestBuildstore(t *testing.T) { db := mustConnectTest() - bs := NewBuildstore(db) + bs := NewJobstore(db) cs := NewCommitstore(db) defer db.Close() g := goblin.Goblin(t) - g.Describe("Buildstore", func() { + g.Describe("Jobstore", func() { // before each test we purge the package table data from the database. g.BeforeEach(func() { - db.Exec("DELETE FROM builds") + db.Exec("DELETE FROM jobs") db.Exec("DELETE FROM commits") }) - g.It("Should Set a build", func() { - build := &types.Build{ - CommitID: 1, - State: "pending", + g.It("Should Set a job", func() { + job := &types.Job{ + BuildID: 1, + Status: "pending", ExitCode: 0, - Sequence: 1, + Number: 1, } - err1 := bs.AddBuild(build) + err1 := bs.AddJob(job) g.Assert(err1 == nil).IsTrue() - g.Assert(build.ID != 0).IsTrue() + g.Assert(job.ID != 0).IsTrue() - build.State = "started" - err2 := bs.SetBuild(build) + job.Status = "started" + err2 := bs.SetJob(job) g.Assert(err2 == nil).IsTrue() - getbuild, err3 := bs.Build(build.ID) + getjob, err3 := bs.Job(job.ID) g.Assert(err3 == nil).IsTrue() - g.Assert(getbuild.State).Equal(build.State) + g.Assert(getjob.Status).Equal(job.Status) }) - g.It("Should Get a Build by ID", func() { - build := &types.Build{ - CommitID: 1, - State: "pending", + g.It("Should Get a Job by ID", func() { + job := &types.Job{ + BuildID: 1, + Status: "pending", ExitCode: 1, - Sequence: 1, + Number: 1, Environment: map[string]string{"foo": "bar"}, } - err1 := bs.AddBuild(build) + err1 := bs.AddJob(job) g.Assert(err1 == nil).IsTrue() - g.Assert(build.ID != 0).IsTrue() + g.Assert(job.ID != 0).IsTrue() - getbuild, err2 := bs.Build(build.ID) + getjob, err2 := bs.Job(job.ID) g.Assert(err2 == nil).IsTrue() - g.Assert(getbuild.ID).Equal(build.ID) - g.Assert(getbuild.State).Equal(build.State) - g.Assert(getbuild.ExitCode).Equal(build.ExitCode) - g.Assert(getbuild.Environment).Equal(build.Environment) - g.Assert(getbuild.Environment["foo"]).Equal("bar") + g.Assert(getjob.ID).Equal(job.ID) + g.Assert(getjob.Status).Equal(job.Status) + g.Assert(getjob.ExitCode).Equal(job.ExitCode) + g.Assert(getjob.Environment).Equal(job.Environment) + g.Assert(getjob.Environment["foo"]).Equal("bar") }) - g.It("Should Get a Build by Sequence", func() { - build := &types.Build{ - CommitID: 1, - State: "pending", + g.It("Should Get a Job by Number", func() { + job := &types.Job{ + BuildID: 1, + Status: "pending", ExitCode: 1, - Sequence: 1, + Number: 1, } - err1 := bs.AddBuild(build) + err1 := bs.AddJob(job) g.Assert(err1 == nil).IsTrue() - g.Assert(build.ID != 0).IsTrue() + g.Assert(job.ID != 0).IsTrue() - getbuild, err2 := bs.BuildSeq(&types.Commit{ID: 1}, 1) + getjob, err2 := bs.JobNumber(&types.Commit{ID: 1}, 1) g.Assert(err2 == nil).IsTrue() - g.Assert(getbuild.ID).Equal(build.ID) - g.Assert(getbuild.State).Equal(build.State) + g.Assert(getjob.ID).Equal(job.ID) + g.Assert(getjob.Status).Equal(job.Status) }) - g.It("Should Get a List of Builds by Commit", func() { - //Add repo - buildList := []*types.Build{ - &types.Build{ - CommitID: 1, - State: "success", - ExitCode: 0, - Sequence: 1, - }, - &types.Build{ - CommitID: 3, - State: "error", - ExitCode: 1, - Sequence: 2, - }, - &types.Build{ - CommitID: 5, - State: "pending", - ExitCode: 0, - Sequence: 3, - }, - } + g.It("Should Get a List of Jobs by Commit", func() { + //In order for buid to be populated, //The AddCommit command will insert builds //if the Commit.Builds array is populated //Add Commit. - commit1 := types.Commit{ + commit := types.Commit{ RepoID: 1, State: types.StateSuccess, Ref: "refs/heads/master", Sha: "14710626f22791619d3b7e9ccf58b10374e5b76d", - Builds: buildList, + Builds: []*types.Job{ + &types.Job{ + BuildID: 1, + Status: "success", + ExitCode: 0, + Number: 1, + }, + &types.Job{ + BuildID: 3, + Status: "error", + ExitCode: 1, + Number: 2, + }, + &types.Job{ + BuildID: 5, + Status: "pending", + ExitCode: 0, + Number: 3, + }, + }, } // - err1 := cs.AddCommit(&commit1) + err1 := cs.AddCommit(&commit) g.Assert(err1 == nil).IsTrue() - bldList, err2 := bs.BuildList(&commit1) + getjobs, err2 := bs.JobList(&commit) g.Assert(err2 == nil).IsTrue() - g.Assert(len(bldList)).Equal(3) - g.Assert(bldList[0].Sequence).Equal(1) - g.Assert(bldList[0].State).Equal(types.StateSuccess) + g.Assert(len(getjobs)).Equal(3) + g.Assert(getjobs[0].Number).Equal(1) + g.Assert(getjobs[0].Status).Equal(types.StateSuccess) }) }) } diff --git a/pkg/store/builtin/commit.go b/pkg/store/builtin/commit.go index da369871..9079866f 100644 --- a/pkg/store/builtin/commit.go +++ b/pkg/store/builtin/commit.go @@ -60,11 +60,9 @@ func (db *Commitstore) AddCommit(commit *types.Commit) error { return err } - for _, build := range commit.Builds { - build.CommitID = commit.ID - build.Created = commit.Created - build.Updated = commit.Updated - err := createBuild(tx, rebind(stmtBuildInsert), build) + for _, job := range commit.Builds { + job.BuildID = commit.ID + err := createJob(tx, rebind(stmtJobInsert), job) if err != nil { return err } @@ -86,9 +84,8 @@ func (db *Commitstore) SetCommit(commit *types.Commit) error { return err } - for _, build := range commit.Builds { - build.Updated = commit.Updated - err = updateBuild(tx, rebind(stmtBuildUpdate), build) + for _, job := range commit.Builds { + err = updateJob(tx, rebind(stmtJobUpdate), job) if err != nil { return err } @@ -138,8 +135,8 @@ WHERE commit_state IN ('pending', 'running'); // SQL statement to cancel all running commits. const buildKillStmt = ` -UPDATE builds SET build_state = 'killed' -WHERE build_state IN ('pending', 'running'); +UPDATE jobs SET job_status = 'killed' +WHERE job_status IN ('pending', 'running'); ` // SQL statement to retrieve the commit number for diff --git a/pkg/store/builtin/migrate/migrate.go b/pkg/store/builtin/migrate/migrate.go index ffea5012..c35a3b52 100644 --- a/pkg/store/builtin/migrate/migrate.go +++ b/pkg/store/builtin/migrate/migrate.go @@ -132,24 +132,21 @@ CREATE INDEX tokens_user_idx ON tokens (token_user_id); ` var buildTable = ` -CREATE TABLE IF NOT EXISTS builds ( - build_id INTEGER PRIMARY KEY AUTOINCREMENT - ,build_commit_id INTEGER - ,build_sequence INTEGER - ,build_state VARCHAR(255) - ,build_exit_code INTEGER - ,build_duration INTEGER - ,build_started INTEGER - ,build_finished INTEGER - ,build_created INTEGER - ,build_updated INTEGER - ,build_environment VARCHAR(2000) - ,UNIQUE(build_commit_id, build_sequence) +CREATE TABLE IF NOT EXISTS jobs ( + job_id INTEGER PRIMARY KEY AUTOINCREMENT + ,job_build_id INTEGER + ,job_number INTEGER + ,job_status VARCHAR(255) + ,job_exit_code INTEGER + ,job_started INTEGER + ,job_finished INTEGER + ,job_environment VARCHAR(2000) + ,UNIQUE(job_build_id, job_number) ); ` var buildCommitIndex = ` -CREATE INDEX builds_commit_idx ON builds (build_commit_id); +CREATE INDEX ix_job_build_id ON jobs (job_build_id); ` var statusTable = ` diff --git a/pkg/store/builtin/store.go b/pkg/store/builtin/store.go index 47a4fff1..975576e6 100644 --- a/pkg/store/builtin/store.go +++ b/pkg/store/builtin/store.go @@ -99,7 +99,7 @@ func New(db *sql.DB) store.Store { *Userstore *Repostore *Commitstore - *Buildstore + *Jobstore *Blobstore *Starstore *Tokenstore @@ -108,7 +108,7 @@ func New(db *sql.DB) store.Store { NewUserstore(db), NewRepostore(db), NewCommitstore(db), - NewBuildstore(db), + NewJobstore(db), NewBlobstore(db), NewStarstore(db), NewTokenstore(db), diff --git a/pkg/store/store.go b/pkg/store/store.go index af5c2332..cc4c6ce2 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -155,16 +155,16 @@ type Store interface { // // Build returns a build by ID. - Build(int64) (*types.Build, error) + Job(int64) (*types.Job, error) - // BuildSeq returns a build by sequence number. - BuildSeq(*types.Commit, int) (*types.Build, error) + // JobNumber returns a jobs by sequence number. + JobNumber(*types.Commit, int) (*types.Job, error) - // BuildList returns a list of all commit builds - BuildList(*types.Commit) ([]*types.Build, error) + // JobList returns a list of all commit jobs + JobList(*types.Commit) ([]*types.Job, error) - // SetBuild updates an existing build. - SetBuild(*types.Build) error + // SetJob updates an existing job. + SetJob(*types.Job) error // diff --git a/pkg/types/build.go b/pkg/types/build.go index 27a8fd83..2cc242b3 100644 --- a/pkg/types/build.go +++ b/pkg/types/build.go @@ -1,19 +1,13 @@ package types -type Build struct { - ID int64 `meddler:"build_id,pk" json:"id"` - CommitID int64 `meddler:"build_commit_id" json:"-" sql:"unique:ux_build_seq,index:ix_build_commit_id"` - State string `meddler:"build_state" json:"state"` - ExitCode int `meddler:"build_exit_code" json:"exit_code"` - Sequence int `meddler:"build_sequence" json:"sequence" sql:"unique:ux_build_seq"` - Duration int64 `meddler:"build_duration" json:"duration"` - Started int64 `meddler:"build_started" json:"started_at"` - Finished int64 `meddler:"build_finished" json:"finished_at"` - Created int64 `meddler:"build_created" json:"created_at"` - Updated int64 `meddler:"build_updated" json:"updated_at"` +type Job struct { + ID int64 `json:"id"` + BuildID int64 `json:"-" sql:"unique:ux_build_number,index:ix_job_build_id"` + Number int `json:"number" sql:"unique:ux_build_number"` + Status string `json:"status"` + ExitCode int `json:"exit_code"` + Started int64 `json:"started_at"` + Finished int64 `json:"finished_at"` - Environment map[string]string `meddler:"build_environment,json" json:"environment" sql:"type:varchar,size:2048"` + Environment map[string]string `json:"environment" sql:"type:varchar,size:2048"` } - -// QUESTION: should we track if it was oom killed? -// OOMKill bool `meddler:"build_oom" json:"oom_kill"` diff --git a/pkg/types/commit.go b/pkg/types/commit.go index 5aa9ed0a..26c6fbc3 100644 --- a/pkg/types/commit.go +++ b/pkg/types/commit.go @@ -30,5 +30,5 @@ type Commit struct { Created int64 `meddler:"commit_created" json:"created_at"` Updated int64 `meddler:"commit_updated" json:"updated_at"` - Builds []*Build `meddler:"-" json:"builds,omitempty" sql:"-"` + Builds []*Job `meddler:"-" json:"builds,omitempty" sql:"-"` }