darkkirb/harness-drone
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

refactor job to proc

Browse source
This commit is contained in:
Brad Rydzewski 2017-04-02 23:13:26 +09:00
parent f090a2d34f
commit ec88661d9e
14 changed files with 496 additions and 544 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

62
model/build.go
View file

@ -2,37 +2,37 @@ package model
// swagger:model build
type Build struct {
ID int64 `json:"id" meddler:"build_id,pk"`
RepoID int64 `json:"-" meddler:"build_repo_id"`
Number int `json:"number" meddler:"build_number"`
Parent int `json:"parent" meddler:"build_parent"`
Event string `json:"event" meddler:"build_event"`
Status string `json:"status" meddler:"build_status"`
Error string `json:"error" meddler:"build_error"`
Enqueued int64 `json:"enqueued_at" meddler:"build_enqueued"`
Created int64 `json:"created_at" meddler:"build_created"`
Started int64 `json:"started_at" meddler:"build_started"`
Finished int64 `json:"finished_at" meddler:"build_finished"`
Deploy string `json:"deploy_to" meddler:"build_deploy"`
Commit string `json:"commit" meddler:"build_commit"`
Branch string `json:"branch" meddler:"build_branch"`
Ref string `json:"ref" meddler:"build_ref"`
Refspec string `json:"refspec" meddler:"build_refspec"`
Remote string `json:"remote" meddler:"build_remote"`
Title string `json:"title" meddler:"build_title"`
Message string `json:"message" meddler:"build_message"`
Timestamp int64 `json:"timestamp" meddler:"build_timestamp"`
Sender string `json:"sender" meddler:"build_sender"`
Author string `json:"author" meddler:"build_author"`
Avatar string `json:"author_avatar" meddler:"build_avatar"`
Email string `json:"author_email" meddler:"build_email"`
Link string `json:"link_url" meddler:"build_link"`
Signed bool `json:"signed" meddler:"build_signed"` // deprecate
Verified bool `json:"verified" meddler:"build_verified"` // deprecate
Reviewer string `json:"reviewed_by" meddler:"build_reviewer"`
Reviewed int64 `json:"reviewed_at" meddler:"build_reviewed"`
Jobs []*Job `json:"jobs,omitempty" meddler:"-"`
Procs []*Proc `json:"procs,omitempty" meddler:"-"`
ID int64 `json:"id" meddler:"build_id,pk"`
RepoID int64 `json:"-" meddler:"build_repo_id"`
Number int `json:"number" meddler:"build_number"`
Parent int `json:"parent" meddler:"build_parent"`
Event string `json:"event" meddler:"build_event"`
Status string `json:"status" meddler:"build_status"`
Error string `json:"error" meddler:"build_error"`
Enqueued int64 `json:"enqueued_at" meddler:"build_enqueued"`
Created int64 `json:"created_at" meddler:"build_created"`
Started int64 `json:"started_at" meddler:"build_started"`
Finished int64 `json:"finished_at" meddler:"build_finished"`
Deploy string `json:"deploy_to" meddler:"build_deploy"`
Commit string `json:"commit" meddler:"build_commit"`
Branch string `json:"branch" meddler:"build_branch"`
Ref string `json:"ref" meddler:"build_ref"`
Refspec string `json:"refspec" meddler:"build_refspec"`
Remote string `json:"remote" meddler:"build_remote"`
Title string `json:"title" meddler:"build_title"`
Message string `json:"message" meddler:"build_message"`
Timestamp int64 `json:"timestamp" meddler:"build_timestamp"`
Sender string `json:"sender" meddler:"build_sender"`
Author string `json:"author" meddler:"build_author"`
Avatar string `json:"author_avatar" meddler:"build_avatar"`
Email string `json:"author_email" meddler:"build_email"`
Link string `json:"link_url" meddler:"build_link"`
Signed bool `json:"signed" meddler:"build_signed"` // deprecate
Verified bool `json:"verified" meddler:"build_verified"` // deprecate
Reviewer string `json:"reviewed_by" meddler:"build_reviewer"`
Reviewed int64 `json:"reviewed_at" meddler:"build_reviewed"`
// Jobs []*Job `json:"jobs,omitempty" meddler:"-"`
Procs []*Proc `json:"procs,omitempty" meddler:"-"`
}
type BuildGroup struct {

21
model/event.go
View file

@ -15,24 +15,5 @@ type Event struct {
Type EventType `json:"type"`
Repo Repo `json:"repo"`
Build Build `json:"build"`
Job Job `json:"job"`
}
// NewEvent creates a new Event for the build, using copies of
// the build data to avoid possible mutation or race conditions.
func NewEvent(t EventType, r *Repo, b *Build, j *Job) *Event {
return &Event{
Type: t,
Repo: *r,
Build: *b,
Job: *j,
}
}
func NewBuildEvent(t EventType, r *Repo, b *Build) *Event {
return &Event{
Type: t,
Repo: *r,
Build: *b,
}
Proc Proc `json:"proc"`
}

30
model/job.go
View file

@ -1,17 +1,17 @@
package model
// swagger:model job
type Job struct {
ID int64 `json:"id" meddler:"job_id,pk"`
BuildID int64 `json:"-" meddler:"job_build_id"`
NodeID int64 `json:"-" meddler:"job_node_id"`
Number int `json:"number" meddler:"job_number"`
Error string `json:"error" meddler:"job_error"`
Status string `json:"status" meddler:"job_status"`
ExitCode int `json:"exit_code" meddler:"job_exit_code"`
Enqueued int64 `json:"enqueued_at" meddler:"job_enqueued"`
Started int64 `json:"started_at" meddler:"job_started"`
Finished int64 `json:"finished_at" meddler:"job_finished"`
Environment map[string]string `json:"environment" meddler:"job_environment,json"`
}
// // swagger:model job
// type Job struct {
// ID int64 `json:"id" meddler:"job_id,pk"`
// BuildID int64 `json:"-" meddler:"job_build_id"`
// NodeID int64 `json:"-" meddler:"job_node_id"`
// Number int `json:"number" meddler:"job_number"`
// Error string `json:"error" meddler:"job_error"`
// Status string `json:"status" meddler:"job_status"`
// ExitCode int `json:"exit_code" meddler:"job_exit_code"`
// Enqueued int64 `json:"enqueued_at" meddler:"job_enqueued"`
// Started int64 `json:"started_at" meddler:"job_started"`
// Finished int64 `json:"finished_at" meddler:"job_finished"`
//
// Environment map[string]string `json:"environment" meddler:"job_environment,json"`
// }

10
model/log.go
View file

@ -1,7 +1,7 @@
package model
type Log struct {
ID int64 `meddler:"log_id,pk"`
JobID int64 `meddler:"log_job_id"`
Data []byte `meddler:"log_data"`
}
// type Log struct {
// ID int64 `meddler:"log_id,pk"`
// JobID int64 `meddler:"log_job_id"`
// Data []byte `meddler:"log_data"`
// }

145
server/build.go
View file

@ -1,7 +1,6 @@
package server
import (
"bufio"
"context"
"encoding/json"
"fmt"
@ -51,14 +50,10 @@ func GetBuild(c *gin.Context) {
c.AbortWithError(http.StatusInternalServerError, err)
return
}
jobs, _ := store.GetJobList(c, build)
procs, _ := store.FromContext(c).ProcList(build)
build.Procs = model.Tree(procs)
out := struct {
*model.Build
Jobs []*model.Job `json:"jobs"`
}{build, jobs}
c.JSON(http.StatusOK, &out)
c.JSON(http.StatusOK, build)
}
func GetBuildLast(c *gin.Context) {
@ -70,23 +65,15 @@ func GetBuildLast(c *gin.Context) {
c.String(http.StatusInternalServerError, err.Error())
return
}
jobs, _ := store.GetJobList(c, build)
out := struct {
*model.Build
Jobs []*model.Job `json:"jobs"`
}{build, jobs}
c.JSON(http.StatusOK, &out)
procs, _ := store.FromContext(c).ProcList(build)
build.Procs = model.Tree(procs)
c.JSON(http.StatusOK, build)
}
func GetBuildLogs(c *gin.Context) {
repo := session.Repo(c)
// the user may specify to stream the full logs,
// or partial logs, capped at 2MB.
full, _ := strconv.ParseBool(c.DefaultQuery("full", "false"))
// parse the build number and job sequence number from
// the repquest parameter.
num, _ := strconv.Atoi(c.Params.ByName("number"))
@ -98,25 +85,22 @@ func GetBuildLogs(c *gin.Context) {
return
}
job, err := store.GetJobNumber(c, build, seq)
proc, err := store.FromContext(c).ProcFind(build, seq)
if err != nil {
c.AbortWithError(404, err)
return
}
r, err := store.ReadLog(c, job)
rc, err := store.FromContext(c).FileRead(proc, "logs.json")
if err != nil {
c.AbortWithError(404, err)
return
}
defer r.Close()
if full {
// TODO implement limited streaming to avoid crashing the browser
}
defer rc.Close()
c.Header("Content-Type", "application/json")
copyLogs(c.Writer, r)
io.Copy(c.Writer, rc)
}
func DeleteBuild(c *gin.Context) {
@ -133,26 +117,27 @@ func DeleteBuild(c *gin.Context) {
return
}
job, err := store.GetJobNumber(c, build, seq)
proc, err := store.FromContext(c).ProcFind(build, seq)
if err != nil {
c.AbortWithError(404, err)
return
}
if job.Status != model.StatusRunning {
if proc.State != model.StatusRunning {
c.String(400, "Cannot cancel a non-running build")
return
}
job.Status = model.StatusKilled
job.Finished = time.Now().Unix()
if job.Started == 0 {
job.Started = job.Finished
proc.State = model.StatusKilled
proc.Stopped = time.Now().Unix()
if proc.Started == 0 {
proc.Started = proc.Stopped
}
job.ExitCode = 137
store.UpdateBuildJob(c, build, job)
proc.ExitCode = 137
// TODO cancel child procs
store.FromContext(c).ProcUpdate(proc)
config.queue.Error(context.Background(), fmt.Sprint(job.ID), queue.ErrCancel)
config.queue.Error(context.Background(), fmt.Sprint(proc.ID), queue.ErrCancel)
c.String(204, "")
}
@ -243,11 +228,31 @@ func PostApproval(c *gin.Context) {
return
}
var pcounter = len(items)
for _, item := range items {
build.Jobs = append(build.Jobs, item.Job)
store.CreateJob(c, item.Job)
// TODO err
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
}
store.FromContext(c).ProcCreate(build.Procs)
//
// publish topic
@ -271,7 +276,7 @@ func PostApproval(c *gin.Context) {
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Job.ID)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
@ -279,7 +284,7 @@ func PostApproval(c *gin.Context) {
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Job.ID),
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
@ -336,23 +341,6 @@ func GetBuildQueue(c *gin.Context) {
c.JSON(200, out)
}
// copyLogs copies the stream from the source to the destination in valid JSON
// format. This converts the logs, which are per-line JSON objects, to a
// proper JSON array.
func copyLogs(dest io.Writer, src io.Reader) error {
io.WriteString(dest, "[")
scanner := bufio.NewScanner(src)
for scanner.Scan() {
io.WriteString(dest, scanner.Text())
io.WriteString(dest, ",\n")
}
io.WriteString(dest, "{}]")
return nil
}
//
//
//
@ -411,9 +399,9 @@ func PostBuild(c *gin.Context) {
return
}
jobs, err := store.GetJobList(c, build)
procs, err := store.FromContext(c).ProcList(build)
if err != nil {
logrus.Errorf("failure to get build %d jobs. %s", build.Number, err)
logrus.Errorf("failure to get build %d procs. %s", build.Number, err)
c.AbortWithError(404, err)
return
}
@ -430,11 +418,11 @@ func PostBuild(c *gin.Context) {
build.ID = 0
build.Number = 0
build.Parent = num
for _, job := range jobs {
job.ID = 0
job.NodeID = 0
for _, proc := range procs {
proc.ID = 0
proc.BuildID = 0
}
err := store.CreateBuild(c, build, jobs...)
err := store.CreateBuild(c, build, procs...)
if err != nil {
c.String(500, err.Error())
return
@ -469,18 +457,17 @@ func PostBuild(c *gin.Context) {
build.Finished = 0
build.Enqueued = time.Now().UTC().Unix()
build.Error = ""
for _, job := range jobs {
for _, proc := range procs {
for k, v := range buildParams {
job.Environment[k] = v
proc.Environ[k] = v
}
job.Error = ""
job.Status = model.StatusPending
job.Started = 0
job.Finished = 0
job.ExitCode = 0
job.NodeID = 0
job.Enqueued = build.Enqueued
store.UpdateJob(c, job)
proc.Error = ""
proc.State = model.StatusPending
proc.Started = 0
proc.Stopped = 0
proc.ExitCode = 0
proc.Machine = ""
store.FromContext(c).ProcUpdate(proc)
}
err = store.UpdateBuild(c, build)
@ -519,9 +506,11 @@ func PostBuild(c *gin.Context) {
for i, item := range items {
// TODO prevent possible index out of bounds
item.Job.ID = jobs[i].ID
build.Jobs = append(build.Jobs, item.Job)
store.UpdateJob(c, item.Job)
item.Proc.ID = procs[i].ID
build.Procs = append(build.Procs, item.Proc)
store.FromContext(c).ProcUpdate(item.Proc)
// TODO update child procs too!
}
//
@ -546,7 +535,7 @@ func PostBuild(c *gin.Context) {
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Job.ID)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
@ -554,7 +543,7 @@ func PostBuild(c *gin.Context) {
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Job.ID),
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})

63
server/hook.go
View file

@ -123,7 +123,7 @@ func PostHook(c *gin.Context) {
// if the remote has a refresh token, the current access token
// may be stale. Therefore, we should refresh prior to dispatching
// the job.
// the build.
if refresher, ok := remote_.(remote.Refresher); ok {
ok, _ := refresher.Refresh(user)
if ok {
@ -221,7 +221,7 @@ func PostHook(c *gin.Context) {
build.RepoID = repo.ID
build.Verified = true
if err := store.CreateBuild(c, build, build.Jobs...); err != nil {
if err := store.CreateBuild(c, build, build.Procs...); err != nil {
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
@ -268,11 +268,32 @@ func PostHook(c *gin.Context) {
return
}
var pcounter = len(items)
for _, item := range items {
build.Jobs = append(build.Jobs, item.Job)
store.CreateJob(c, item.Job)
// TODO err
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
}
store.FromContext(c).ProcCreate(build.Procs)
//
// publish topic
@ -296,7 +317,7 @@ func PostHook(c *gin.Context) {
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Job.ID)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
@ -304,7 +325,7 @@ func PostHook(c *gin.Context) {
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Job.ID),
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
@ -315,7 +336,7 @@ func PostHook(c *gin.Context) {
}
// return the metadata from the cli context.
func metadataFromStruct(repo *model.Repo, build, last *model.Build, job *model.Job, link string) frontend.Metadata {
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
return frontend.Metadata{
Repo: frontend.Repo{
Name: repo.Name,
@ -368,8 +389,8 @@ func metadataFromStruct(repo *model.Repo, build, last *model.Build, job *model.J
},
},
Job: frontend.Job{
Number: job.Number,
Matrix: job.Environment,
Number: proc.PID,
Matrix: proc.Environ,
},
Sys: frontend.System{
Name: "drone",
@ -390,7 +411,7 @@ type builder struct {
}
type buildItem struct {
Job *model.Job
Proc *model.Proc
Platform string
Labels map[string]string
Config *backend.Config
@ -408,15 +429,15 @@ func (b *builder) Build() ([]*buildItem, error) {
var items []*buildItem
for i, axis := range axes {
job := &model.Job{
BuildID: b.Curr.ID,
Number: i + 1,
Status: model.StatusPending,
Environment: axis,
Enqueued: b.Curr.Created,
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
}
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, job, b.Link)
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
@ -481,11 +502,11 @@ func (b *builder) Build() ([]*buildItem, error) {
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
job.ID,
proc.ID,
time.Now().Unix(),
),
),
compiler.WithEnviron(job.Environment),
compiler.WithEnviron(proc.Environ),
compiler.WithProxy(),
// TODO ability to set global volumes for things like certs
compiler.WithVolumes(),
@ -507,7 +528,7 @@ func (b *builder) Build() ([]*buildItem, error) {
}
item := &buildItem{
Job: job,
Proc: proc,
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,

8
server/stream.go
View file

@ -66,13 +66,13 @@ func LogStream(c *gin.Context) {
c.AbortWithError(404, err)
return
}
job, err := store.GetJobNumber(c, build, jobn)
proc, err := store.FromContext(c).ProcFind(build, jobn)
if err != nil {
logrus.Debugln("stream cannot get job number.", err)
logrus.Debugln("stream cannot get proc number.", err)
c.AbortWithError(404, err)
return
}
if job.Status != model.StatusRunning {
if proc.State != model.StatusRunning {
logrus.Debugln("stream not found.")
c.AbortWithStatus(404)
return
@ -102,7 +102,7 @@ func LogStream(c *gin.Context) {
go func() {
// TODO remove global variable
config.logger.Tail(ctx, fmt.Sprint(job.ID), func(entries ...*logging.Entry) {
config.logger.Tail(ctx, fmt.Sprint(proc.ID), func(entries ...*logging.Entry) {
for _, entry := range entries {
select {
case <-ctx.Done():

10
store/datastore/builds.go
View file

@ -55,7 +55,7 @@ func (db *datastore) GetBuildQueue() ([]*model.Feed, error) {
return feed, err
}
func (db *datastore) CreateBuild(build *model.Build, jobs ...*model.Job) error {
func (db *datastore) CreateBuild(build *model.Build, procs ...*model.Proc) error {
var number int
db.QueryRow(rebind(buildNumberLast), build.RepoID).Scan(&number)
build.Number = number + 1
@ -65,11 +65,9 @@ func (db *datastore) CreateBuild(build *model.Build, jobs ...*model.Job) error {
if err != nil {
return err
}
for i, job := range jobs {
job.BuildID = build.ID
job.Number = i + 1
job.Enqueued = build.Created
err = meddler.Insert(db, jobTable, job)
for _, proc := range procs {
proc.BuildID = build.ID
err = meddler.Insert(db, "procs", proc)
if err != nil {
return err
}

36
store/datastore/builds_test.go
View file

@ -28,7 +28,7 @@ func TestBuilds(t *testing.T) {
Status: model.StatusSuccess,
Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac",
}
err := s.CreateBuild(&build, []*model.Job{}...)
err := s.CreateBuild(&build, []*model.Proc{}...)
g.Assert(err == nil).IsTrue()
g.Assert(build.ID != 0).IsTrue()
g.Assert(build.Number).Equal(1)
@ -42,7 +42,7 @@ func TestBuilds(t *testing.T) {
Status: model.StatusSuccess,
Commit: "85f8c029b902ed9400bc600bac301a0aadb144ac",
}
s.CreateBuild(&build, []*model.Job{}...)
s.CreateBuild(&build, []*model.Proc{}...)
build.Status = model.StatusRunning
err1 := s.UpdateBuild(&build)
getbuild, err2 := s.GetBuild(build.ID)
@ -59,7 +59,7 @@ func TestBuilds(t *testing.T) {
RepoID: 1,
Status: model.StatusSuccess,
}
s.CreateBuild(&build, []*model.Job{}...)
s.CreateBuild(&build, []*model.Proc{}...)
getbuild, err := s.GetBuild(build.ID)
g.Assert(err == nil).IsTrue()
g.Assert(build.ID).Equal(getbuild.ID)
@ -76,8 +76,8 @@ func TestBuilds(t *testing.T) {
RepoID: 1,
Status: model.StatusPending,
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
getbuild, err3 := s.GetBuildNumber(&model.Repo{ID: 1}, build2.Number)
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -98,8 +98,8 @@ func TestBuilds(t *testing.T) {
Status: model.StatusPending,
Ref: "refs/pull/6",
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
getbuild, err3 := s.GetBuildRef(&model.Repo{ID: 1}, "refs/pull/6")
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -121,8 +121,8 @@ func TestBuilds(t *testing.T) {
Status: model.StatusPending,
Ref: "refs/pull/6",
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
getbuild, err3 := s.GetBuildRef(&model.Repo{ID: 1}, "refs/pull/6")
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -146,8 +146,8 @@ func TestBuilds(t *testing.T) {
Branch: "dev",
Commit: "85f8c029b902ed9400bc600bac301a0aadb144aa",
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
getbuild, err3 := s.GetBuildCommit(&model.Repo{ID: 1}, build2.Commit, build2.Branch)
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -174,8 +174,8 @@ func TestBuilds(t *testing.T) {
Commit: "85f8c029b902ed9400bc600bac301a0aadb144aa",
Event: model.EventPush,
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
getbuild, err3 := s.GetBuildLast(&model.Repo{ID: 1}, build2.Branch)
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -207,9 +207,9 @@ func TestBuilds(t *testing.T) {
Branch: "master",
Commit: "85f8c029b902ed9400bc600bac301a0aadb144aa",
}
err1 := s.CreateBuild(build1, []*model.Job{}...)
err2 := s.CreateBuild(build2, []*model.Job{}...)
err3 := s.CreateBuild(build3, []*model.Job{}...)
err1 := s.CreateBuild(build1, []*model.Proc{}...)
err2 := s.CreateBuild(build2, []*model.Proc{}...)
err3 := s.CreateBuild(build3, []*model.Proc{}...)
getbuild, err4 := s.GetBuildLastBefore(&model.Repo{ID: 1}, build3.Branch, build3.ID)
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
@ -232,8 +232,8 @@ func TestBuilds(t *testing.T) {
RepoID: 1,
Status: model.StatusSuccess,
}
s.CreateBuild(build1, []*model.Job{}...)
s.CreateBuild(build2, []*model.Job{}...)
s.CreateBuild(build1, []*model.Proc{}...)
s.CreateBuild(build2, []*model.Proc{}...)
builds, err := s.GetBuildList(&model.Repo{ID: 1})
g.Assert(err == nil).IsTrue()
g.Assert(len(builds)).Equal(2)

95
store/datastore/jobs.go
View file

@ -1,49 +1,50 @@
package datastore
import (
"github.com/drone/drone/model"
"github.com/russross/meddler"
)
func (db *datastore) GetJob(id int64) (*model.Job, error) {
var job = new(model.Job)
var err = meddler.Load(db, jobTable, job, id)
return job, err
}
func (db *datastore) GetJobNumber(build *model.Build, num int) (*model.Job, error) {
var job = new(model.Job)
var err = meddler.QueryRow(db, job, rebind(jobNumberQuery), build.ID, num)
return job, err
}
func (db *datastore) GetJobList(build *model.Build) ([]*model.Job, error) {
var jobs = []*model.Job{}
var err = meddler.QueryAll(db, &jobs, rebind(jobListQuery), build.ID)
return jobs, err
}
func (db *datastore) CreateJob(job *model.Job) error {
return meddler.Insert(db, jobTable, job)
}
func (db *datastore) UpdateJob(job *model.Job) error {
return meddler.Update(db, jobTable, job)
}
const jobTable = "jobs"
const jobListQuery = `
SELECT *
FROM jobs
WHERE job_build_id = ?
ORDER BY job_number ASC
`
const jobNumberQuery = `
SELECT *
FROM jobs
WHERE job_build_id = ?
AND job_number = ?
LIMIT 1
`
//
// import (
// "github.com/drone/drone/model"
// "github.com/russross/meddler"
// )
//
// func (db *datastore) GetJob(id int64) (*model.Job, error) {
// var job = new(model.Job)
// var err = meddler.Load(db, jobTable, job, id)
// return job, err
// }
//
// func (db *datastore) GetJobNumber(build *model.Build, num int) (*model.Job, error) {
// var job = new(model.Job)
// var err = meddler.QueryRow(db, job, rebind(jobNumberQuery), build.ID, num)
// return job, err
// }
//
// func (db *datastore) GetJobList(build *model.Build) ([]*model.Job, error) {
// var jobs = []*model.Job{}
// var err = meddler.QueryAll(db, &jobs, rebind(jobListQuery), build.ID)
// return jobs, err
// }
//
// func (db *datastore) CreateJob(job *model.Job) error {
// return meddler.Insert(db, jobTable, job)
// }
//
// func (db *datastore) UpdateJob(job *model.Job) error {
// return meddler.Update(db, jobTable, job)
// }
//
// const jobTable = "jobs"
//
// const jobListQuery = `
// SELECT *
// FROM jobs
// WHERE job_build_id = ?
// ORDER BY job_number ASC
// `
//
// const jobNumberQuery = `
// SELECT *
// FROM jobs
// WHERE job_build_id = ?
// AND job_number = ?
// LIMIT 1
// `

233
store/datastore/jobs_test.go
View file

@ -1,118 +1,119 @@
package datastore
import (
"testing"
"github.com/drone/drone/model"
"github.com/franela/goblin"
)
func TestJobs(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
g := goblin.Goblin(t)
g.Describe("Job", func() {
// before each test we purge the package table data from the database.
g.BeforeEach(func() {
db.Exec("DELETE FROM jobs")
db.Exec("DELETE FROM builds")
})
g.It("Should Set a job", func() {
job := &model.Job{
BuildID: 1,
Status: "pending",
ExitCode: 0,
Number: 1,
}
err1 := s.CreateJob(job)
g.Assert(err1 == nil).IsTrue()
g.Assert(job.ID != 0).IsTrue()
job.Status = "started"
err2 := s.UpdateJob(job)
g.Assert(err2 == nil).IsTrue()
getjob, err3 := s.GetJob(job.ID)
g.Assert(err3 == nil).IsTrue()
g.Assert(getjob.Status).Equal(job.Status)
})
g.It("Should Get a Job by ID", func() {
job := &model.Job{
BuildID: 1,
Status: "pending",
ExitCode: 1,
Number: 1,
Environment: map[string]string{"foo": "bar"},
}
err1 := s.CreateJob(job)
g.Assert(err1 == nil).IsTrue()
g.Assert(job.ID != 0).IsTrue()
getjob, err2 := s.GetJob(job.ID)
g.Assert(err2 == nil).IsTrue()
g.Assert(getjob.ID).Equal(job.ID)
g.Assert(getjob.Status).Equal(job.Status)
g.Assert(getjob.ExitCode).Equal(job.ExitCode)
g.Assert(getjob.Environment).Equal(job.Environment)
g.Assert(getjob.Environment["foo"]).Equal("bar")
})
g.It("Should Get a Job by Number", func() {
job := &model.Job{
BuildID: 1,
Status: "pending",
ExitCode: 1,
Number: 1,
}
err1 := s.CreateJob(job)
g.Assert(err1 == nil).IsTrue()
g.Assert(job.ID != 0).IsTrue()
getjob, err2 := s.GetJobNumber(&model.Build{ID: 1}, 1)
g.Assert(err2 == nil).IsTrue()
g.Assert(getjob.ID).Equal(job.ID)
g.Assert(getjob.Status).Equal(job.Status)
})
g.It("Should Get a List of Jobs by Commit", func() {
build := model.Build{
RepoID: 1,
Status: model.StatusSuccess,
}
jobs := []*model.Job{
{
BuildID: 1,
Status: "success",
ExitCode: 0,
Number: 1,
},
{
BuildID: 3,
Status: "error",
ExitCode: 1,
Number: 2,
},
{
BuildID: 5,
Status: "pending",
ExitCode: 0,
Number: 3,
},
}
err1 := s.CreateBuild(&build, jobs...)
g.Assert(err1 == nil).IsTrue()
getjobs, err2 := s.GetJobList(&build)
g.Assert(err2 == nil).IsTrue()
g.Assert(len(getjobs)).Equal(3)
g.Assert(getjobs[0].Number).Equal(1)
g.Assert(getjobs[0].Status).Equal(model.StatusSuccess)
})
})
}
//
// import (
// "testing"
//
// "github.com/drone/drone/model"
// "github.com/franela/goblin"
// )
//
// func TestJobs(t *testing.T) {
// db := openTest()
// defer db.Close()
//
// s := From(db)
// g := goblin.Goblin(t)
// g.Describe("Job", func() {
//
// // before each test we purge the package table data from the database.
// g.BeforeEach(func() {
// db.Exec("DELETE FROM jobs")
// db.Exec("DELETE FROM builds")
// })
//
// g.It("Should Set a job", func() {
// job := &model.Job{
// BuildID: 1,
// Status: "pending",
// ExitCode: 0,
// Number: 1,
// }
// err1 := s.CreateJob(job)
// g.Assert(err1 == nil).IsTrue()
// g.Assert(job.ID != 0).IsTrue()
//
// job.Status = "started"
// err2 := s.UpdateJob(job)
// g.Assert(err2 == nil).IsTrue()
//
// getjob, err3 := s.GetJob(job.ID)
// g.Assert(err3 == nil).IsTrue()
// g.Assert(getjob.Status).Equal(job.Status)
// })
//
// g.It("Should Get a Job by ID", func() {
// job := &model.Job{
// BuildID: 1,
// Status: "pending",
// ExitCode: 1,
// Number: 1,
// Environment: map[string]string{"foo": "bar"},
// }
// err1 := s.CreateJob(job)
// g.Assert(err1 == nil).IsTrue()
// g.Assert(job.ID != 0).IsTrue()
//
// getjob, err2 := s.GetJob(job.ID)
// g.Assert(err2 == nil).IsTrue()
// g.Assert(getjob.ID).Equal(job.ID)
// g.Assert(getjob.Status).Equal(job.Status)
// g.Assert(getjob.ExitCode).Equal(job.ExitCode)
// g.Assert(getjob.Environment).Equal(job.Environment)
// g.Assert(getjob.Environment["foo"]).Equal("bar")
// })
//
// g.It("Should Get a Job by Number", func() {
// job := &model.Job{
// BuildID: 1,
// Status: "pending",
// ExitCode: 1,
// Number: 1,
// }
// err1 := s.CreateJob(job)
// g.Assert(err1 == nil).IsTrue()
// g.Assert(job.ID != 0).IsTrue()
//
// getjob, err2 := s.GetJobNumber(&model.Build{ID: 1}, 1)
// g.Assert(err2 == nil).IsTrue()
// g.Assert(getjob.ID).Equal(job.ID)
// g.Assert(getjob.Status).Equal(job.Status)
// })
//
// g.It("Should Get a List of Jobs by Commit", func() {
//
// build := model.Build{
// RepoID: 1,
// Status: model.StatusSuccess,
// }
// jobs := []*model.Job{
// {
// BuildID: 1,
// Status: "success",
// ExitCode: 0,
// Number: 1,
// },
// {
// BuildID: 3,
// Status: "error",
// ExitCode: 1,
// Number: 2,
// },
// {
// BuildID: 5,
// Status: "pending",
// ExitCode: 0,
// Number: 3,
// },
// }
//
// err1 := s.CreateBuild(&build, jobs...)
// g.Assert(err1 == nil).IsTrue()
// getjobs, err2 := s.GetJobList(&build)
// g.Assert(err2 == nil).IsTrue()
// g.Assert(len(getjobs)).Equal(3)
// g.Assert(getjobs[0].Number).Equal(1)
// g.Assert(getjobs[0].Status).Equal(model.StatusSuccess)
// })
// })
// }

69
store/datastore/logs.go
View file

@ -1,36 +1,37 @@
package datastore
import (
"bytes"
"io"
"io/ioutil"
"github.com/drone/drone/model"
"github.com/russross/meddler"
)
func (db *datastore) ReadLog(job *model.Job) (io.ReadCloser, error) {
var log = new(model.Log)
var err = meddler.QueryRow(db, log, rebind(logQuery), job.ID)
var buf = bytes.NewBuffer(log.Data)
return ioutil.NopCloser(buf), err
}
func (db *datastore) WriteLog(job *model.Job, r io.Reader) error {
var log = new(model.Log)
var err = meddler.QueryRow(db, log, rebind(logQuery), job.ID)
if err != nil {
log = &model.Log{JobID: job.ID}
}
log.Data, _ = ioutil.ReadAll(r)
return meddler.Save(db, logTable, log)
}
const logTable = "logs"
const logQuery = `
SELECT *
FROM logs
WHERE log_job_id=?
LIMIT 1
`
//
// import (
// "bytes"
// "io"
// "io/ioutil"
//
// "github.com/drone/drone/model"
// "github.com/russross/meddler"
// )
//
// func (db *datastore) ReadLog(job *model.Job) (io.ReadCloser, error) {
// var log = new(model.Log)
// var err = meddler.QueryRow(db, log, rebind(logQuery), job.ID)
// var buf = bytes.NewBuffer(log.Data)
// return ioutil.NopCloser(buf), err
// }
//
// func (db *datastore) WriteLog(job *model.Job, r io.Reader) error {
// var log = new(model.Log)
// var err = meddler.QueryRow(db, log, rebind(logQuery), job.ID)
// if err != nil {
// log = &model.Log{JobID: job.ID}
// }
// log.Data, _ = ioutil.ReadAll(r)
// return meddler.Save(db, logTable, log)
// }
//
// const logTable = "logs"
//
// const logQuery = `
// SELECT *
// FROM logs
// WHERE log_job_id=?
// LIMIT 1
// `

117
store/datastore/logs_test.go
View file

@ -1,60 +1,61 @@
package datastore
import (
"bytes"
"io/ioutil"
"testing"
"github.com/drone/drone/model"
"github.com/franela/goblin"
)
func TestLogs(t *testing.T) {
db := openTest()
defer db.Close()
s := From(db)
g := goblin.Goblin(t)
g.Describe("Logs", func() {
// before each test be sure to purge the package
// table data from the database.
g.BeforeEach(func() {
db.Exec("DELETE FROM logs")
})
g.It("Should create a log", func() {
job := model.Job{
ID: 1,
}
buf := bytes.NewBufferString("echo hi")
err := s.WriteLog(&job, buf)
g.Assert(err == nil).IsTrue()
rc, err := s.ReadLog(&job)
g.Assert(err == nil).IsTrue()
defer rc.Close()
out, _ := ioutil.ReadAll(rc)
g.Assert(string(out)).Equal("echo hi")
})
g.It("Should update a log", func() {
job := model.Job{
ID: 1,
}
buf1 := bytes.NewBufferString("echo hi")
buf2 := bytes.NewBufferString("echo allo?")
err1 := s.WriteLog(&job, buf1)
err2 := s.WriteLog(&job, buf2)
g.Assert(err1 == nil).IsTrue()
g.Assert(err2 == nil).IsTrue()
rc, err := s.ReadLog(&job)
g.Assert(err == nil).IsTrue()
defer rc.Close()
out, _ := ioutil.ReadAll(rc)
g.Assert(string(out)).Equal("echo allo?")
})
})
}
//
// import (
// "bytes"
// "io/ioutil"
// "testing"
//
// "github.com/drone/drone/model"
// "github.com/franela/goblin"
// )
//
// func TestLogs(t *testing.T) {
// db := openTest()
// defer db.Close()
//
// s := From(db)
// g := goblin.Goblin(t)
// g.Describe("Logs", func() {
//
// // before each test be sure to purge the package
// // table data from the database.
// g.BeforeEach(func() {
// db.Exec("DELETE FROM logs")
// })
//
// g.It("Should create a log", func() {
// job := model.Job{
// ID: 1,
// }
// buf := bytes.NewBufferString("echo hi")
// err := s.WriteLog(&job, buf)
// g.Assert(err == nil).IsTrue()
//
// rc, err := s.ReadLog(&job)
// g.Assert(err == nil).IsTrue()
// defer rc.Close()
// out, _ := ioutil.ReadAll(rc)
// g.Assert(string(out)).Equal("echo hi")
// })
//
// g.It("Should update a log", func() {
// job := model.Job{
// ID: 1,
// }
// buf1 := bytes.NewBufferString("echo hi")
// buf2 := bytes.NewBufferString("echo allo?")
// err1 := s.WriteLog(&job, buf1)
// err2 := s.WriteLog(&job, buf2)
// g.Assert(err1 == nil).IsTrue()
// g.Assert(err2 == nil).IsTrue()
//
// rc, err := s.ReadLog(&job)
// g.Assert(err == nil).IsTrue()
// defer rc.Close()
// out, _ := ioutil.ReadAll(rc)
// g.Assert(string(out)).Equal("echo allo?")
// })
//
// })
// }

141
store/store.go
View file

@ -107,31 +107,31 @@ type Store interface {
GetBuildQueue() ([]*model.Feed, error)
// CreateBuild creates a new build and jobs.
CreateBuild(*model.Build, ...*model.Job) error
CreateBuild(*model.Build, ...*model.Proc) error
// UpdateBuild updates a build.
UpdateBuild(*model.Build) error
// GetJob gets a job by unique ID.
GetJob(int64) (*model.Job, error)
// GetJobNumber gets a job by number.
GetJobNumber(*model.Build, int) (*model.Job, error)
// GetJobList gets a list of all users in the system.
GetJobList(*model.Build) ([]*model.Job, error)
// CreateJob creates a job.
CreateJob(*model.Job) error
// UpdateJob updates a job.
UpdateJob(*model.Job) error
// ReadLog reads the Job logs from the datastore.
ReadLog(*model.Job) (io.ReadCloser, error)
// WriteLog writes the job logs to the datastore.
WriteLog(*model.Job, io.Reader) error
// // GetJob gets a job by unique ID.
// GetJob(int64) (*model.Job, error)
//
// // GetJobNumber gets a job by number.
// GetJobNumber(*model.Build, int) (*model.Job, error)
//
// // GetJobList gets a list of all users in the system.
// GetJobList(*model.Build) ([]*model.Job, error)
//
// // CreateJob creates a job.
// CreateJob(*model.Job) error
//
// // UpdateJob updates a job.
// UpdateJob(*model.Job) error
//
// // ReadLog reads the Job logs from the datastore.
// ReadLog(*model.Job) (io.ReadCloser, error)
//
// // WriteLog writes the job logs to the datastore.
// WriteLog(*model.Job, io.Reader) error
GetAgent(int64) (*model.Agent, error)
@ -348,82 +348,41 @@ func GetBuildQueue(c context.Context) ([]*model.Feed, error) {
return FromContext(c).GetBuildQueue()
}
func CreateBuild(c context.Context, build *model.Build, jobs ...*model.Job) error {
return FromContext(c).CreateBuild(build, jobs...)
func CreateBuild(c context.Context, build *model.Build, procs ...*model.Proc) error {
return FromContext(c).CreateBuild(build, procs...)
}
func UpdateBuild(c context.Context, build *model.Build) error {
return FromContext(c).UpdateBuild(build)
}
func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool, error) {
if err := UpdateJob(c, job); err != nil {
return false, err
}
// if the job is running or started we don't need to update the build
// status since.
if job.Status == model.StatusRunning || job.Status == model.StatusPending {
return false, nil
}
jobs, err := GetJobList(c, build)
if err != nil {
return false, err
}
// check to see if all jobs are finished for this build. If yes, we need to
// calcualte the overall build status and finish time.
status := model.StatusSuccess
finish := job.Finished
for _, job := range jobs {
if job.Finished > finish {
finish = job.Finished
}
switch job.Status {
case model.StatusSuccess:
// no-op
case model.StatusRunning, model.StatusPending:
return false, nil
default:
status = job.Status
}
}
build.Status = status
build.Finished = finish
if err := FromContext(c).UpdateBuild(build); err != nil {
return false, err
}
return true, nil
}
func GetJob(c context.Context, id int64) (*model.Job, error) {
return FromContext(c).GetJob(id)
}
func GetJobNumber(c context.Context, build *model.Build, num int) (*model.Job, error) {
return FromContext(c).GetJobNumber(build, num)
}
func GetJobList(c context.Context, build *model.Build) ([]*model.Job, error) {
return FromContext(c).GetJobList(build)
}
func CreateJob(c context.Context, job *model.Job) error {
return FromContext(c).CreateJob(job)
}
func UpdateJob(c context.Context, job *model.Job) error {
return FromContext(c).UpdateJob(job)
}
func ReadLog(c context.Context, job *model.Job) (io.ReadCloser, error) {
return FromContext(c).ReadLog(job)
}
func WriteLog(c context.Context, job *model.Job, r io.Reader) error {
return FromContext(c).WriteLog(job, r)
}
// func GetJob(c context.Context, id int64) (*model.Job, error) {
// return FromContext(c).GetJob(id)
// }
//
// func GetJobNumber(c context.Context, build *model.Build, num int) (*model.Job, error) {
// return FromContext(c).GetJobNumber(build, num)
// }
//
// func GetJobList(c context.Context, build *model.Build) ([]*model.Job, error) {
// return FromContext(c).GetJobList(build)
// }
//
// func CreateJob(c context.Context, job *model.Job) error {
// return FromContext(c).CreateJob(job)
// }
//
// func UpdateJob(c context.Context, job *model.Job) error {
// return FromContext(c).UpdateJob(job)
// }
//
// func ReadLog(c context.Context, job *model.Job) (io.ReadCloser, error) {
// return FromContext(c).ReadLog(job)
// }
//
// func WriteLog(c context.Context, job *model.Job, r io.Reader) error {
// return FromContext(c).WriteLog(job, r)
// }
func GetAgent(c context.Context, id int64) (*model.Agent, error) {
return FromContext(c).GetAgent(id)