diff --git a/pkg/database/builds.go b/pkg/database/builds.go index fefe1e7b..d3fc4f72 100644 --- a/pkg/database/builds.go +++ b/pkg/database/builds.go @@ -10,7 +10,7 @@ const buildTable = "builds" // SQL Queries to retrieve a list of all Commits belonging to a Repo. const buildStmt = ` -SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout +SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout, buildscript FROM builds WHERE commit_id = ? ORDER BY slug ASC @@ -18,7 +18,7 @@ ORDER BY slug ASC // SQL Queries to retrieve a Build by id. const buildFindStmt = ` -SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout +SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout, buildscript FROM builds WHERE id = ? LIMIT 1 @@ -26,7 +26,7 @@ LIMIT 1 // SQL Queries to retrieve a Commit by name and repo id. const buildFindSlugStmt = ` -SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout +SELECT id, commit_id, slug, status, started, finished, duration, created, updated, stdout, buildscript FROM builds WHERE slug = ? AND commit_id = ? LIMIT 1 diff --git a/pkg/database/migrate/20140522205400_save_drone_yml.go b/pkg/database/migrate/20140522205400_save_drone_yml.go new file mode 100644 index 00000000..443dc716 --- /dev/null +++ b/pkg/database/migrate/20140522205400_save_drone_yml.go @@ -0,0 +1,19 @@ +package migrate + +type rev20140522205400 struct{} + +var SaveDroneYml = &rev20140522205400{} + +func (r *rev20140522205400) Revision() int64 { + return 20140522205400 +} + +func (r *rev20140522205400) Up(mg *MigrationDriver) error { + _, err := mg.AddColumn("builds", "buildscript TEXT") + return err +} + +func (r *rev20140522205400) Down(mg *MigrationDriver) error { + _, err := mg.DropColumns("builds", "buildscript") + return err +} diff --git a/pkg/database/migrate/all.go b/pkg/database/migrate/all.go index c745a4ed..14c6a9a7 100644 --- a/pkg/database/migrate/all.go +++ b/pkg/database/migrate/all.go @@ -13,6 +13,7 @@ func (m *Migration) All() *Migration { m.Add(GitHubEnterpriseSupport) m.Add(AddOpenInvitationColumn) m.Add(AddGitlabColumns) + m.Add(SaveDroneYml) // m.Add(...) // ... diff --git a/pkg/handler/bitbucket.go b/pkg/handler/bitbucket.go index 48ab7f2c..be88727b 100644 --- a/pkg/handler/bitbucket.go +++ b/pkg/handler/bitbucket.go @@ -5,7 +5,6 @@ import ( "net/http" "time" - "github.com/drone/drone/pkg/build/script" "github.com/drone/drone/pkg/database" . "github.com/drone/drone/pkg/model" "github.com/drone/drone/pkg/queue" @@ -83,16 +82,6 @@ func (h *BitbucketHandler) Hook(w http.ResponseWriter, r *http.Request) error { return RenderText(w, http.StatusText(http.StatusNotFound), http.StatusNotFound) } - // parse the build script - buildscript, err := script.ParseBuild([]byte(raw.Data), repo.Params) - if err != nil { - msg := "Could not parse your .drone.yml file. It needs to be a valid drone yaml file.\n\n" + err.Error() + "\n" - if err := saveFailedBuild(commit, msg); err != nil { - return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - } - // save the commit to the database if err := database.SaveCommit(commit); err != nil { return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -104,12 +93,13 @@ func (h *BitbucketHandler) Hook(w http.ResponseWriter, r *http.Request) error { build.CommitID = commit.ID build.Created = time.Now().UTC() build.Status = "Pending" + build.BuildScript = raw.Data if err := database.SaveBuild(build); err != nil { return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } // send the build to the queue - h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build, Script: buildscript}) + h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build}) // OK! return RenderText(w, http.StatusText(http.StatusOK), http.StatusOK) diff --git a/pkg/handler/github.go b/pkg/handler/github.go index 14baf5c5..0e0237ec 100644 --- a/pkg/handler/github.go +++ b/pkg/handler/github.go @@ -6,7 +6,6 @@ import ( "strconv" "time" - "github.com/drone/drone/pkg/build/script" "github.com/drone/drone/pkg/database" . "github.com/drone/drone/pkg/model" "github.com/drone/drone/pkg/queue" @@ -131,22 +130,8 @@ func (h *GithubHandler) Hook(w http.ResponseWriter, r *http.Request) error { } // decode the content. Note: Not sure this will ever happen...it basically means a GitHub API issue - raw, err := content.DecodeContent() + buildscript, err := content.DecodeContent() if err != nil { - msg := "Could not decode the yaml from GitHub. Check that your .drone.yml is a valid yaml file.\n" - if err := saveFailedBuild(commit, msg); err != nil { - return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - } - - // parse the build script - buildscript, err := script.ParseBuild(raw, repo.Params) - if err != nil { - msg := "Could not parse your .drone.yml file. It needs to be a valid drone yaml file.\n\n" + err.Error() + "\n" - if err := saveFailedBuild(commit, msg); err != nil { - return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } return RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) } @@ -161,6 +146,7 @@ func (h *GithubHandler) Hook(w http.ResponseWriter, r *http.Request) error { build.CommitID = commit.ID build.Created = time.Now().UTC() build.Status = "Pending" + build.BuildScript = string(buildscript) if err := database.SaveBuild(build); err != nil { return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } @@ -169,7 +155,7 @@ func (h *GithubHandler) Hook(w http.ResponseWriter, r *http.Request) error { //realtime.CommitPending(repo.UserID, repo.TeamID, repo.ID, commit.ID, repo.Private) //realtime.BuildPending(repo.UserID, repo.TeamID, repo.ID, commit.ID, build.ID, repo.Private) - h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build, Script: buildscript}) //Push(repo, commit, build, buildscript) + h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build}) //Push(repo, commit, build) // OK! return RenderText(w, http.StatusText(http.StatusOK), http.StatusOK) @@ -250,22 +236,12 @@ func (h *GithubHandler) PullRequestHook(w http.ResponseWriter, r *http.Request) } // decode the content - raw, err := content.DecodeContent() + buildscript, err := content.DecodeContent() if err != nil { RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) return } - // parse the build script - buildscript, err := script.ParseBuild(raw, repo.Params) - if err != nil { - // TODO if the YAML is invalid we should create a commit record - // with an ERROR status so that the user knows why a build wasn't - // triggered in the system - RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - return - } - // save the commit to the database if err := database.SaveCommit(commit); err != nil { RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -278,6 +254,7 @@ func (h *GithubHandler) PullRequestHook(w http.ResponseWriter, r *http.Request) build.CommitID = commit.ID build.Created = time.Now().UTC() build.Status = "Pending" + build.BuildScript = string(buildscript) if err := database.SaveBuild(build); err != nil { RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) return @@ -285,7 +262,7 @@ func (h *GithubHandler) PullRequestHook(w http.ResponseWriter, r *http.Request) // notify websocket that a new build is pending // TODO we should, for consistency, just put this inside Queue.Add() - h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build, Script: buildscript}) + h.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build}) // OK! RenderText(w, http.StatusText(http.StatusOK), http.StatusOK) diff --git a/pkg/handler/gitlab.go b/pkg/handler/gitlab.go index f22c9d88..ae0e6310 100644 --- a/pkg/handler/gitlab.go +++ b/pkg/handler/gitlab.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/drone/drone/pkg/build/script" "github.com/drone/drone/pkg/database" . "github.com/drone/drone/pkg/model" "github.com/drone/drone/pkg/queue" @@ -207,7 +206,7 @@ func (g *GitlabHandler) Hook(w http.ResponseWriter, r *http.Request) error { // get the drone.yml file from GitHub client := gogitlab.NewGitlab(settings.GitlabApiUrl, g.apiPath, user.GitlabToken) - content, err := client.RepoRawFile(ns(repo.Owner, repo.Name), commit.Hash, ".drone.yml") + buildscript, err := client.RepoRawFile(ns(repo.Owner, repo.Name), commit.Hash, ".drone.yml") if err != nil { msg := "No .drone.yml was found in this repository. You need to add one.\n" if err := saveFailedBuild(commit, msg); err != nil { @@ -216,16 +215,6 @@ func (g *GitlabHandler) Hook(w http.ResponseWriter, r *http.Request) error { return RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) } - // parse the build script - buildscript, err := script.ParseBuild(content, repo.Params) - if err != nil { - msg := "Could not parse your .drone.yml file. It needs to be a valid drone yaml file.\n\n" + err.Error() + "\n" - if err := saveFailedBuild(commit, msg); err != nil { - return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - } - return RenderText(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) - } - // save the commit to the database if err := database.SaveCommit(commit); err != nil { return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) @@ -237,11 +226,12 @@ func (g *GitlabHandler) Hook(w http.ResponseWriter, r *http.Request) error { build.CommitID = commit.ID build.Created = time.Now().UTC() build.Status = "Pending" + build.BuildScript = string(buildscript) if err := database.SaveBuild(build); err != nil { return RenderText(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) } - g.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build, Script: buildscript}) + g.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build}) // OK! return RenderText(w, http.StatusText(http.StatusOK), http.StatusOK) @@ -287,7 +277,7 @@ func (g *GitlabHandler) PullRequestHook(p *gogitlab.HookPayload, repo *Repo, use commit.Timestamp = src.Commit.AuthoredDateRaw commit.SetAuthor(src.Commit.Author.Email) - content, err := client.RepoRawFile(strconv.Itoa(obj.SourceProjectId), commit.Hash, ".drone.yml") + buildscript, err := client.RepoRawFile(strconv.Itoa(obj.SourceProjectId), commit.Hash, ".drone.yml") if err != nil { msg := "No .drone.yml was found in this repository. You need to add one.\n" if err := saveFailedBuild(commit, msg); err != nil { @@ -296,16 +286,6 @@ func (g *GitlabHandler) PullRequestHook(p *gogitlab.HookPayload, repo *Repo, use return fmt.Errorf("Error to fetch build script: %q", err) } - // parse the build script - buildscript, err := script.ParseBuild(content, repo.Params) - if err != nil { - msg := "Could not parse your .drone.yml file. It needs to be a valid drone yaml file.\n\n" + err.Error() + "\n" - if err := saveFailedBuild(commit, msg); err != nil { - return fmt.Errorf("Failed to save build: %q", err) - } - return fmt.Errorf("Failed to parse build script: %q", err) - } - // save the commit to the database if err := database.SaveCommit(commit); err != nil { return fmt.Errorf("Failed to save commit: %q", err) @@ -317,11 +297,12 @@ func (g *GitlabHandler) PullRequestHook(p *gogitlab.HookPayload, repo *Repo, use build.CommitID = commit.ID build.Created = time.Now().UTC() build.Status = "Pending" + build.BuildScript = string(buildscript) if err := database.SaveBuild(build); err != nil { return fmt.Errorf("Failed to save build: %q", err) } - g.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build, Script: buildscript}) + g.queue.Add(&queue.BuildTask{Repo: repo, Commit: commit, Build: build}) return nil } diff --git a/pkg/handler/testing/handler_test.go b/pkg/handler/testing/handler_test.go index 9ea71824..0616d961 100644 --- a/pkg/handler/testing/handler_test.go +++ b/pkg/handler/testing/handler_test.go @@ -45,7 +45,7 @@ func TestRepoHandler(t *testing.T) { So(rec.Code, ShouldEqual, 303) }) Convey("Private repo can not be viewed by a non team member", func() { - req, err := http.NewRequest("GET", "/github.com/drone/drone", nil) + req, err := http.NewRequest("GET", "/github.com/drone/drone", nil) So(err, ShouldBeNil) rec := httptest.NewRecorder() setUserSession(rec, req, "rick@el.to.ro") @@ -61,7 +61,7 @@ func dummyUserRepo(w http.ResponseWriter, r *http.Request, u *User, repo *Repo) func setUserSession(w http.ResponseWriter, r *http.Request, username string) { handler.SetCookie(w, r, "_sess", username) - resp := http.Response{Header: w.Header()} + resp := http.Response{Header: w.Header()} for _, v := range resp.Cookies() { r.AddCookie(v) } diff --git a/pkg/model/build.go b/pkg/model/build.go index 44864156..6dd0e2b6 100644 --- a/pkg/model/build.go +++ b/pkg/model/build.go @@ -15,16 +15,17 @@ const ( ) type Build struct { - ID int64 `meddler:"id,pk" json:"id"` - CommitID int64 `meddler:"commit_id" json:"-"` - Slug string `meddler:"slug" json:"slug"` - Status string `meddler:"status" json:"status"` - Started time.Time `meddler:"started,utctime" json:"started"` - Finished time.Time `meddler:"finished,utctime" json:"finished"` - Duration int64 `meddler:"duration" json:"duration"` - Created time.Time `meddler:"created,utctime" json:"created"` - Updated time.Time `meddler:"updated,utctime" json:"updated"` - Stdout string `meddler:"stdout" json:"-"` + ID int64 `meddler:"id,pk" json:"id"` + CommitID int64 `meddler:"commit_id" json:"-"` + Slug string `meddler:"slug" json:"slug"` + Status string `meddler:"status" json:"status"` + Started time.Time `meddler:"started,utctime" json:"started"` + Finished time.Time `meddler:"finished,utctime" json:"finished"` + Duration int64 `meddler:"duration" json:"duration"` + Created time.Time `meddler:"created,utctime" json:"created"` + Updated time.Time `meddler:"updated,utctime" json:"updated"` + Stdout string `meddler:"stdout" json:"-"` + BuildScript string `meddler:"buildscript" json:"-"` } // HumanDuration returns a human-readable approximation of a duration diff --git a/pkg/queue/queue.go b/pkg/queue/queue.go index cb7cfba9..58743463 100644 --- a/pkg/queue/queue.go +++ b/pkg/queue/queue.go @@ -1,7 +1,6 @@ package queue import ( - "github.com/drone/drone/pkg/build/script" . "github.com/drone/drone/pkg/model" ) @@ -16,10 +15,6 @@ type BuildTask struct { Repo *Repo Commit *Commit Build *Build - - // Build instructions from the .drone.yml - // file, unmarshalled. - Script *script.Build } // Start N workers with the given build runner. diff --git a/pkg/queue/worker.go b/pkg/queue/worker.go index bf9aeab3..71adb15b 100644 --- a/pkg/queue/worker.go +++ b/pkg/queue/worker.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/drone/drone/pkg/build/git" r "github.com/drone/drone/pkg/build/repo" + "github.com/drone/drone/pkg/build/script" "github.com/drone/drone/pkg/channel" "github.com/drone/drone/pkg/database" . "github.com/drone/drone/pkg/model" @@ -83,9 +84,16 @@ func (w *worker) execute(task *BuildTask) error { Host: settings.URL().String(), } + // parse the build script + buildscript, err := script.ParseBuild([]byte(task.Build.BuildScript), task.Repo.Params) + if err != nil { + log.Printf("Could not parse your .drone.yml file. It needs to be a valid drone yaml file.\n\n" + err.Error() + "\n") + return err + } + // send all "started" notifications - if task.Script.Notifications != nil { - task.Script.Notifications.Send(context) + if buildscript.Notifications != nil { + buildscript.Notifications.Send(context) } // Send "started" notification to Github @@ -113,7 +121,7 @@ func (w *worker) execute(task *BuildTask) error { // this is not a pull request (for security purposes) if task.Repo.Params != nil && len(task.Commit.PullRequest) == 0 { for k, v := range task.Repo.Params { - task.Script.Env = append(task.Script.Env, k+"="+v) + buildscript.Env = append(buildscript.Env, k+"="+v) } } @@ -126,7 +134,7 @@ func (w *worker) execute(task *BuildTask) error { }() // execute the build - passed, buildErr := w.runBuild(task, buf) + passed, buildErr := w.runBuild(task, buildscript, buf) task.Build.Finished = time.Now().UTC() task.Commit.Finished = time.Now().UTC() @@ -162,14 +170,14 @@ func (w *worker) execute(task *BuildTask) error { channel.Close(consoleslug) // send all "finished" notifications - if task.Script.Notifications != nil { - task.Script.Notifications.Send(context) + if buildscript.Notifications != nil { + buildscript.Notifications.Send(context) } return nil } -func (w *worker) runBuild(task *BuildTask, buf io.Writer) (bool, error) { +func (w *worker) runBuild(task *BuildTask, buildscript *script.Build, buf io.Writer) (bool, error) { repo := &r.Repo{ Name: task.Repo.Slug, Path: task.Repo.URL, @@ -177,11 +185,11 @@ func (w *worker) runBuild(task *BuildTask, buf io.Writer) (bool, error) { Commit: task.Commit.Hash, PR: task.Commit.PullRequest, Dir: filepath.Join("/var/cache/drone/src", task.Repo.Slug), - Depth: git.GitDepth(task.Script.Git), + Depth: git.GitDepth(buildscript.Git), } return w.runner.Run( - task.Script, + buildscript, repo, []byte(task.Repo.PrivateKey), task.Repo.Privileged,