From e319aaff155aa87e015c8cf06420042e8759fedd Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 16:49:27 +0800 Subject: [PATCH 1/6] add reviewer fields, endpoints --- model/build.go | 7 +++- model/const.go | 16 ++++---- model/repo.go | 1 + remote/bitbucket/convert.go | 30 +++++++++----- remote/github/convert.go | 23 +++++++---- remote/github/convert_test.go | 2 + remote/github/fixtures/hooks.go | 2 +- remote/gitlab/gitlab.go | 12 ++++-- remote/gogs/fixtures/hooks.go | 1 + remote/gogs/helper.go | 15 +++++++ remote/gogs/types.go | 1 + router/router.go | 2 + server/build.go | 63 +++++++++++++++++++++++++++++ store/datastore/ddl/mysql/12.sql | 18 +++++++++ store/datastore/ddl/postgres/12.sql | 18 +++++++++ store/datastore/ddl/sqlite3/12.sql | 18 +++++++++ 16 files changed, 198 insertions(+), 31 deletions(-) create mode 100644 store/datastore/ddl/mysql/12.sql create mode 100644 store/datastore/ddl/postgres/12.sql create mode 100644 store/datastore/ddl/sqlite3/12.sql diff --git a/model/build.go b/model/build.go index 3f7e5952..56c2f69a 100644 --- a/model/build.go +++ b/model/build.go @@ -22,12 +22,15 @@ type Build struct { Title string `json:"title" meddler:"build_title"` Message string `json:"message" meddler:"build_message"` Timestamp int64 `json:"timestamp" meddler:"build_timestamp"` + Sender string `json:"sender" meddler:"build_sender"` Author string `json:"author" meddler:"build_author"` Avatar string `json:"author_avatar" meddler:"build_avatar"` Email string `json:"author_email" meddler:"build_email"` Link string `json:"link_url" meddler:"build_link"` - Signed bool `json:"signed" meddler:"build_signed"` - Verified bool `json:"verified" meddler:"build_verified"` + Signed bool `json:"signed" meddler:"build_signed"` // deprecate + Verified bool `json:"verified" meddler:"build_verified"` // deprecate + Reviewer string `json:"reviewed_by" meddler:"build_reviewer"` + Reviewed int64 `json:"reviewed_at" meddler:"build_reviewed"` Jobs []*Job `json:"jobs,omitempty" meddler:"-"` } diff --git a/model/const.go b/model/const.go index efac6b3b..dfc2f308 100644 --- a/model/const.go +++ b/model/const.go @@ -8,13 +8,15 @@ const ( ) const ( - StatusSkipped = "skipped" - StatusPending = "pending" - StatusRunning = "running" - StatusSuccess = "success" - StatusFailure = "failure" - StatusKilled = "killed" - StatusError = "error" + StatusSkipped = "skipped" + StatusPending = "pending" + StatusRunning = "running" + StatusSuccess = "success" + StatusFailure = "failure" + StatusKilled = "killed" + StatusError = "error" + StatusBlocked = "blocked" + StatusDeclined = "declined" ) const ( diff --git a/model/repo.go b/model/repo.go index 6baaf81f..7e881b96 100644 --- a/model/repo.go +++ b/model/repo.go @@ -29,5 +29,6 @@ type Repo struct { AllowPush bool `json:"allow_push" meddler:"repo_allow_push"` AllowDeploy bool `json:"allow_deploys" meddler:"repo_allow_deploys"` AllowTag bool `json:"allow_tags" meddler:"repo_allow_tags"` + Config string `json:"config_path" meddler:"repo_config_path"` Hash string `json:"-" meddler:"repo_hash"` } diff --git a/remote/bitbucket/convert.go b/remote/bitbucket/convert.go index bc6d9171..b5272b40 100644 --- a/remote/bitbucket/convert.go +++ b/remote/bitbucket/convert.go @@ -2,8 +2,8 @@ package bitbucket import ( "fmt" - "regexp" "net/url" + "regexp" "strings" "github.com/drone/drone/model" @@ -19,17 +19,19 @@ const ( ) const ( - descPending = "this build is pending" - descSuccess = "the build was successful" - descFailure = "the build failed" - descError = "oops, something went wrong" + descPending = "this build is pending" + descSuccess = "the build was successful" + descFailure = "the build failed" + descBlocked = "the build requires approval" + descDeclined = "the build was rejected" + descError = "oops, something went wrong" ) // convertStatus is a helper function used to convert a Drone status to a // Bitbucket commit status. func convertStatus(status string) string { switch status { - case model.StatusPending, model.StatusRunning: + case model.StatusPending, model.StatusRunning, model.StatusBlocked: return statusPending case model.StatusSuccess: return statusSuccess @@ -48,6 +50,10 @@ func convertDesc(status string) string { return descSuccess case model.StatusFailure: return descFailure + case model.StatusBlocked: + return descBlocked + case model.StatusDeclined: + return descDeclined default: return descError } @@ -163,6 +169,7 @@ func convertPullHook(from *internal.PullRequestHook) *model.Build { Message: from.PullRequest.Desc, Avatar: from.Actor.Links.Avatar.Href, Author: from.Actor.Login, + Sender: from.Actor.Login, Timestamp: from.PullRequest.Updated.UTC().Unix(), } } @@ -177,6 +184,7 @@ func convertPushHook(hook *internal.PushHook, change *internal.Change) *model.Bu Message: change.New.Target.Message, Avatar: hook.Actor.Links.Avatar.Href, Author: hook.Actor.Login, + Sender: hook.Actor.Login, Timestamp: change.New.Target.Date.UTC().Unix(), } switch change.New.Type { @@ -198,9 +206,9 @@ var reGitMail = regexp.MustCompile("<(.*)>") // extracts the email from a git commit author string func extractEmail(gitauthor string) (author string) { - matches := reGitMail.FindAllStringSubmatch(gitauthor,-1) - if len(matches) == 1 { - author = matches[0][1] - } - return + matches := reGitMail.FindAllStringSubmatch(gitauthor, -1) + if len(matches) == 1 { + author = matches[0][1] + } + return } diff --git a/remote/github/convert.go b/remote/github/convert.go index 84ec72e1..ade90038 100644 --- a/remote/github/convert.go +++ b/remote/github/convert.go @@ -19,10 +19,12 @@ const ( ) const ( - descPending = "this build is pending" - descSuccess = "the build was successful" - descFailure = "the build failed" - descError = "oops, something went wrong" + descPending = "this build is pending" + descSuccess = "the build was successful" + descFailure = "the build failed" + descBlocked = "the build requires approval" + descDeclined = "the build was rejected" + descError = "oops, something went wrong" ) const ( @@ -35,12 +37,12 @@ const ( // GitHub commit status. func convertStatus(status string) string { switch status { - case model.StatusPending, model.StatusRunning: + case model.StatusPending, model.StatusRunning, model.StatusBlocked: return statusPending + case model.StatusFailure, model.StatusDeclined: + return statusFailure case model.StatusSuccess: return statusSuccess - case model.StatusFailure: - return statusFailure default: return statusError } @@ -56,6 +58,10 @@ func convertDesc(status string) string { return descSuccess case model.StatusFailure: return descFailure + case model.StatusBlocked: + return descBlocked + case model.StatusDeclined: + return descDeclined default: return descError } @@ -185,6 +191,7 @@ func convertPushHook(from *webhook) *model.Build { Avatar: from.Sender.Avatar, Author: from.Sender.Login, Remote: from.Repo.CloneURL, + Sender: from.Sender.Login, } if len(build.Author) == 0 { build.Author = from.Head.Author.Username @@ -213,6 +220,7 @@ func convertDeployHook(from *webhook) *model.Build { Ref: from.Deployment.Ref, Branch: from.Deployment.Ref, Deploy: from.Deployment.Env, + Sender: from.Sender.Login, } // if the ref is a sha or short sha we need to manuallyconstruct the ref. if strings.HasPrefix(build.Commit, build.Ref) || build.Commit == build.Ref { @@ -242,6 +250,7 @@ func convertPullHook(from *webhook, merge bool) *model.Build { Author: from.PullRequest.User.Login, Avatar: from.PullRequest.User.Avatar, Title: from.PullRequest.Title, + Sender: from.Sender.Login, Remote: from.PullRequest.Head.Repo.CloneURL, Refspec: fmt.Sprintf(refspec, from.PullRequest.Head.Ref, diff --git a/remote/github/convert_test.go b/remote/github/convert_test.go index dd8ea80f..3fc8b8e4 100644 --- a/remote/github/convert_test.go +++ b/remote/github/convert_test.go @@ -181,6 +181,7 @@ func Test_helper(t *testing.T) { from.PullRequest.Title = "Updated README.md" from.PullRequest.User.Login = "octocat" from.PullRequest.User.Avatar = "https://avatars1.githubusercontent.com/u/583231" + from.Sender.Login = "octocat" build := convertPullHook(from, true) g.Assert(build.Event).Equal(model.EventPull) @@ -193,6 +194,7 @@ func Test_helper(t *testing.T) { g.Assert(build.Title).Equal(from.PullRequest.Title) g.Assert(build.Author).Equal(from.PullRequest.User.Login) g.Assert(build.Avatar).Equal(from.PullRequest.User.Avatar) + g.Assert(build.Sender).Equal(from.Sender.Login) }) g.It("should convert a deployment from webhook", func() { diff --git a/remote/github/fixtures/hooks.go b/remote/github/fixtures/hooks.go index f8dbf317..da9baac6 100644 --- a/remote/github/fixtures/hooks.go +++ b/remote/github/fixtures/hooks.go @@ -95,7 +95,7 @@ const HookPullRequest = ` "default_branch": "master" }, "sender": { - "login": "baxterthehacker", + "login": "octocat", "avatar_url": "https://avatars.githubusercontent.com/u/6752317?v=3" } } diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index f1da201a..728d8d8e 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -629,18 +629,20 @@ const ( ) const ( - DescPending = "this build is pending" - DescRunning = "this buils is running" + DescPending = "the build is pending" + DescRunning = "the buils is running" DescSuccess = "the build was successful" DescFailure = "the build failed" DescCanceled = "the build canceled" + DescBlocked = "the build is pending approval" + DescDeclined = "the build was rejected" ) // getStatus is a helper functin that converts a Drone // status to a GitHub status. func getStatus(status string) string { switch status { - case model.StatusPending: + case model.StatusPending, model.StatusBlocked: return StatusPending case model.StatusRunning: return StatusRunning @@ -669,6 +671,10 @@ func getDesc(status string) string { return DescFailure case model.StatusKilled: return DescCanceled + case model.StatusBlocked: + return DescBlocked + case model.StatusDeclined: + return DescDeclined default: return DescFailure } diff --git a/remote/gogs/fixtures/hooks.go b/remote/gogs/fixtures/hooks.go index 0fd5eb19..2ed55601 100644 --- a/remote/gogs/fixtures/hooks.go +++ b/remote/gogs/fixtures/hooks.go @@ -129,6 +129,7 @@ var HookPullRequest = `{ }, "sender": { "id": 1, + "login": "gordon", "username": "gordon", "full_name": "Gordon the Gopher", "email": "gordon@golang.org", diff --git a/remote/gogs/helper.go b/remote/gogs/helper.go index e4f3e138..7e0ef1db 100644 --- a/remote/gogs/helper.go +++ b/remote/gogs/helper.go @@ -74,6 +74,10 @@ func buildFromPush(hook *pushHook) *model.Build { if author == "" { author = hook.Sender.Username } + sender := hook.Sender.Username + if sender == "" { + sender = hook.Sender.Login + } return &model.Build{ Event: model.EventPush, @@ -85,6 +89,7 @@ func buildFromPush(hook *pushHook) *model.Build { Avatar: avatar, Author: author, Timestamp: time.Now().UTC().Unix(), + Sender: sender, } } @@ -98,6 +103,10 @@ func buildFromTag(hook *pushHook) *model.Build { if author == "" { author = hook.Sender.Username } + sender := hook.Sender.Username + if sender == "" { + sender = hook.Sender.Login + } return &model.Build{ Event: model.EventTag, @@ -108,6 +117,7 @@ func buildFromTag(hook *pushHook) *model.Build { Message: fmt.Sprintf("created tag %s", hook.Ref), Avatar: avatar, Author: author, + Sender: sender, Timestamp: time.Now().UTC().Unix(), } } @@ -118,6 +128,10 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build { hook.Repo.URL, fixMalformedAvatar(hook.PullRequest.User.Avatar), ) + sender := hook.Sender.Username + if sender == "" { + sender = hook.Sender.Login + } build := &model.Build{ Event: model.EventPull, Commit: hook.PullRequest.Head.Sha, @@ -127,6 +141,7 @@ func buildFromPullRequest(hook *pullRequestHook) *model.Build { Message: hook.PullRequest.Title, Author: hook.PullRequest.User.Username, Avatar: avatar, + Sender: sender, Title: hook.PullRequest.Title, Refspec: fmt.Sprintf("%s:%s", hook.PullRequest.Head.Ref, diff --git a/remote/gogs/types.go b/remote/gogs/types.go index 0b0949d4..1bf33c62 100644 --- a/remote/gogs/types.go +++ b/remote/gogs/types.go @@ -116,6 +116,7 @@ type pullRequestHook struct { } `json:"repository"` Sender struct { ID int64 `json:"id"` + Login string `json:"login"` Username string `json:"username"` Name string `json:"full_name"` Email string `json:"email"` diff --git a/router/router.go b/router/router.go index 55ecb84e..48123891 100644 --- a/router/router.go +++ b/router/router.go @@ -108,6 +108,8 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { repo.POST("/chown", session.MustRepoAdmin(), server.ChownRepo) repo.POST("/builds/:number", session.MustPush, server.PostBuild) + repo.POST("/builds/:number/approve", session.MustPush, server.PostApproval) + repo.POST("/builds/:number/decline", session.MustPush, server.PostDecline) repo.DELETE("/builds/:number/:job", session.MustPush, server.DeleteBuild) } } diff --git a/server/build.go b/server/build.go index 755c8f17..946d8add 100644 --- a/server/build.go +++ b/server/build.go @@ -156,6 +156,69 @@ func DeleteBuild(c *gin.Context) { c.String(204, "") } +func PostApproval(c *gin.Context) { + var ( + repo = session.Repo(c) + user = session.User(c) + num, _ = strconv.Atoi( + c.Params.ByName("number"), + ) + ) + + build, err := store.GetBuildNumber(c, repo, num) + if err != nil { + c.AbortWithError(404, err) + return + } + if build.Status != model.StatusBlocked { + c.String(500, "cannot decline a build with status %s", build.Status) + return + } + build.Status = model.StatusPending + build.Reviewed = time.Now().Unix() + build.Reviewer = user.Login + + if err := store.UpdateBuild(c, build); err != nil { + c.String(500, "error updating build. %s", err) + return + } + + // + // TODO start build + // + + c.JSON(200, build) +} + +func PostDecline(c *gin.Context) { + var ( + repo = session.Repo(c) + user = session.User(c) + num, _ = strconv.Atoi( + c.Params.ByName("number"), + ) + ) + + build, err := store.GetBuildNumber(c, repo, num) + if err != nil { + c.AbortWithError(404, err) + return + } + if build.Status != model.StatusBlocked { + c.String(500, "cannot decline a build with status %s", build.Status) + return + } + build.Status = model.StatusDeclined + build.Reviewed = time.Now().Unix() + build.Reviewer = user.Login + + if err := store.UpdateBuild(c, build); err != nil { + c.String(500, "error updating build. %s", err) + return + } + c.JSON(200, build) +} + func GetBuildQueue(c *gin.Context) { out, err := store.GetBuildQueue(c) if err != nil { diff --git a/store/datastore/ddl/mysql/12.sql b/store/datastore/ddl/mysql/12.sql new file mode 100644 index 00000000..39db9500 --- /dev/null +++ b/store/datastore/ddl/mysql/12.sql @@ -0,0 +1,18 @@ +-- +migrate Up + +ALTER TABLE repos ADD COLUMN repo_config_path VARCHAR(255); +ALTER TABLE builds ADD COLUMN build_sender VARCHAR(255); +ALTER TABLE builds ADD COLUMN build_reviewer VARCHAR(255); +ALTER TABLE builds ADD COLUMN build_reviewed INTEGER; + +UPDATE repos SET repo_config_path = ''; +UPDATE builds SET build_reviewer = ''; +UPDATE builds SET build_reviewed = 0; +UPDATE builds SET build_sender = ''; + +-- +migrate Down + +ALTER TABLE repos DROP COLUMN repo_config_path; +ALTER TABLE builds DROP COLUMN build_sender; +ALTER TABLE builds DROP COLUMN build_reviewer; +ALTER TABLE builds DROP COLUMN build_reviewed; diff --git a/store/datastore/ddl/postgres/12.sql b/store/datastore/ddl/postgres/12.sql new file mode 100644 index 00000000..d55f2ae5 --- /dev/null +++ b/store/datastore/ddl/postgres/12.sql @@ -0,0 +1,18 @@ +-- +migrate Up + +ALTER TABLE repos ADD COLUMN repo_config_path VARCHAR(255); +ALTER TABLE builds ADD COLUMN build_reviewer VARCHAR(255); +ALTER TABLE builds ADD COLUMN build_reviewed INTEGER; +ALTER TABLE builds ADD COLUMN build_sender VARCHAR(255); + +UPDATE repos SET repo_config_path = ''; +UPDATE builds SET build_reviewer = ''; +UPDATE builds SET build_reviewed = 0; +UPDATE builds SET build_sender = ''; + +-- +migrate Down + +ALTER TABLE repos DROP COLUMN repo_config_path; +ALTER TABLE builds DROP COLUMN build_reviewer; +ALTER TABLE builds DROP COLUMN build_reviewed; +ALTER TABLE builds DROP COLUMN build_sender; diff --git a/store/datastore/ddl/sqlite3/12.sql b/store/datastore/ddl/sqlite3/12.sql new file mode 100644 index 00000000..4392c850 --- /dev/null +++ b/store/datastore/ddl/sqlite3/12.sql @@ -0,0 +1,18 @@ +-- +migrate Up + +ALTER TABLE repos ADD COLUMN repo_config_path TEXT; +ALTER TABLE builds ADD COLUMN build_reviewer TEXT; +ALTER TABLE builds ADD COLUMN build_reviewed INTEGER; +ALTER TABLE builds ADD COLUMN build_sender TEXT; + +UPDATE repos SET repo_config_path = ''; +UPDATE builds SET build_reviewer = ''; +UPDATE builds SET build_reviewed = 0; +UPDATE builds SET build_sender = ''; + +-- +migrate Down + +ALTER TABLE repos DROP COLUMN repo_config_path; +ALTER TABLE builds DROP COLUMN build_reviewer; +ALTER TABLE builds DROP COLUMN build_reviewed; +ALTER TABLE builds DROP COLUMN build_sender; From a4e03defdc83e5d3ec9994a89c930d29b1cec6a0 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 17:05:49 +0800 Subject: [PATCH 2/6] add approve/decline to client/cli --- client/client.go | 6 ++++++ client/client_impl.go | 18 ++++++++++++++++++ drone/build.go | 2 ++ drone/build_approve.go | 39 +++++++++++++++++++++++++++++++++++++++ drone/build_decline.go | 39 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+) create mode 100644 drone/build_approve.go create mode 100644 drone/build_decline.go diff --git a/client/client.go b/client/client.go index 0ed884d9..d959d7b9 100644 --- a/client/client.go +++ b/client/client.go @@ -102,6 +102,12 @@ type Client interface { // the prior history. BuildFork(string, string, int, map[string]string) (*model.Build, error) + // BuildApprove approves a blocked build. + BuildApprove(string, string, int) (*model.Build, error) + + // BuildDecline declines a blocked build. + BuildDecline(string, string, int) (*model.Build, error) + // BuildLogs returns the build logs for the specified job. BuildLogs(string, string, int, int) (io.ReadCloser, error) diff --git a/client/client_impl.go b/client/client_impl.go index 182bc7e0..0f316797 100644 --- a/client/client_impl.go +++ b/client/client_impl.go @@ -34,6 +34,8 @@ const ( pathEncrypt = "%s/api/repos/%s/%s/encrypt" pathBuilds = "%s/api/repos/%s/%s/builds" pathBuild = "%s/api/repos/%s/%s/builds/%v" + pathApprove = "%s/api/repos/%s/%s/builds/%d/approve" + pathDecline = "%s/api/repos/%s/%s/builds/%d/decline" pathJob = "%s/api/repos/%s/%s/builds/%d/%d" pathLog = "%s/api/repos/%s/%s/logs/%d/%d" pathKey = "%s/api/repos/%s/%s/key" @@ -257,6 +259,22 @@ func (c *client) BuildFork(owner, name string, num int, params map[string]string return out, err } +// BuildApprove approves a blocked build. +func (c *client) BuildApprove(owner, name string, num int) (*model.Build, error) { + out := new(model.Build) + uri := fmt.Sprintf(pathApprove, c.base, owner, name, num) + err := c.post(uri, nil, out) + return out, err +} + +// BuildDecline declines a blocked build. +func (c *client) BuildDecline(owner, name string, num int) (*model.Build, error) { + out := new(model.Build) + uri := fmt.Sprintf(pathDecline, c.base, owner, name, num) + err := c.post(uri, nil, out) + return out, err +} + // BuildLogs returns the build logs for the specified job. func (c *client) BuildLogs(owner, name string, num, job int) (io.ReadCloser, error) { uri := fmt.Sprintf(pathLog, c.base, owner, name, num, job) diff --git a/drone/build.go b/drone/build.go index 65c1ece1..78572952 100644 --- a/drone/build.go +++ b/drone/build.go @@ -12,6 +12,8 @@ var buildCmd = cli.Command{ buildInfoCmd, buildStopCmd, buildStartCmd, + buildApproveCmd, + buildDeclineCmd, buildQueueCmd, }, } diff --git a/drone/build_approve.go b/drone/build_approve.go new file mode 100644 index 00000000..a9dfff48 --- /dev/null +++ b/drone/build_approve.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "strconv" + + "github.com/urfave/cli" +) + +var buildApproveCmd = cli.Command{ + Name: "approve", + Usage: "approve a build", + Action: buildApprove, +} + +func buildApprove(c *cli.Context) (err error) { + repo := c.Args().First() + owner, name, err := parseRepo(repo) + if err != nil { + return err + } + number, err := strconv.Atoi(c.Args().Get(1)) + if err != nil { + return err + } + + client, err := newClient(c) + if err != nil { + return err + } + + _, err = client.BuildApprove(owner, name, number) + if err != nil { + return err + } + + fmt.Printf("Approving build %s/%s#%d\n", owner, name, number) + return nil +} diff --git a/drone/build_decline.go b/drone/build_decline.go new file mode 100644 index 00000000..fb4ce35d --- /dev/null +++ b/drone/build_decline.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "strconv" + + "github.com/urfave/cli" +) + +var buildDeclineCmd = cli.Command{ + Name: "decline", + Usage: "decline a build", + Action: buildDecline, +} + +func buildDecline(c *cli.Context) (err error) { + repo := c.Args().First() + owner, name, err := parseRepo(repo) + if err != nil { + return err + } + number, err := strconv.Atoi(c.Args().Get(1)) + if err != nil { + return err + } + + client, err := newClient(c) + if err != nil { + return err + } + + _, err = client.BuildDecline(owner, name, number) + if err != nil { + return err + } + + fmt.Printf("Declining build %s/%s#%d\n", owner, name, number) + return nil +} From bfdd09c48475521c8a9f1e10832233f8d11cc4af Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 17:16:39 +0800 Subject: [PATCH 3/6] update github status on decline --- server/build.go | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/server/build.go b/server/build.go index 946d8add..2f37396a 100644 --- a/server/build.go +++ b/server/build.go @@ -10,7 +10,7 @@ import ( "strconv" "time" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/cncd/pipeline/pipeline/rpc" "github.com/cncd/pubsub" "github.com/cncd/queue" @@ -192,9 +192,10 @@ func PostApproval(c *gin.Context) { func PostDecline(c *gin.Context) { var ( - repo = session.Repo(c) - user = session.User(c) - num, _ = strconv.Atoi( + remote_ = remote.FromContext(c) + repo = session.Repo(c) + user = session.User(c) + num, _ = strconv.Atoi( c.Params.ByName("number"), ) ) @@ -212,10 +213,28 @@ func PostDecline(c *gin.Context) { build.Reviewed = time.Now().Unix() build.Reviewer = user.Login - if err := store.UpdateBuild(c, build); err != nil { + err = store.UpdateBuild(c, build) + if err != nil { c.String(500, "error updating build. %s", err) return } + + owner, err := store.GetUser(c, repo.UserID) + if err == nil { + if refresher, ok := remote_.(remote.Refresher); ok { + ok, _ := refresher.Refresh(user) + if ok { + store.UpdateUser(c, user) + } + } + + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(owner, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) + } + } + c.JSON(200, build) } @@ -266,14 +285,14 @@ func PostBuild(c *gin.Context) { user, err := store.GetUser(c, repo.UserID) if err != nil { - log.Errorf("failure to find repo owner %s. %s", repo.FullName, err) + logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err) c.AbortWithError(500, err) return } build, err := store.GetBuildNumber(c, repo, num) if err != nil { - log.Errorf("failure to get build %d. %s", num, err) + logrus.Errorf("failure to get build %d. %s", num, err) c.AbortWithError(404, err) return } @@ -292,21 +311,21 @@ func PostBuild(c *gin.Context) { cfg := ToConfig(c) raw, err := remote_.File(user, repo, build, cfg.Yaml) if err != nil { - log.Errorf("failure to get build config for %s. %s", repo.FullName, err) + logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) return } netrc, err := remote_.Netrc(user, repo) if err != nil { - log.Errorf("failure to generate netrc for %s. %s", repo.FullName, err) + logrus.Errorf("failure to generate netrc for %s. %s", repo.FullName, err) c.AbortWithError(500, err) return } jobs, err := store.GetJobList(c, build) if err != nil { - log.Errorf("failure to get build %d jobs. %s", build.Number, err) + logrus.Errorf("failure to get build %d jobs. %s", build.Number, err) c.AbortWithError(404, err) return } @@ -389,7 +408,7 @@ func PostBuild(c *gin.Context) { last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) secs, err := store.GetMergedSecretList(c, repo) if err != nil { - log.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) + logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) } b := builder{ From 130c623a355b721e39e9e07d14da183169de8750 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 19:25:53 +0800 Subject: [PATCH 4/6] compare yaml files --- remote/bitbucket/bitbucket.go | 7 +- remote/bitbucketserver/bitbucketserver.go | 15 ++- remote/gerrit/gerrit.go | 5 + remote/github/github.go | 9 +- remote/gitlab/client/project.go | 28 ++++- remote/gitlab/gitlab.go | 15 +++ remote/gogs/gogs.go | 5 + remote/mock/remote.go | 73 ++++++++---- remote/remote.go | 8 +- server/build.go | 134 ++++++++++++++++++---- server/hook.go | 59 +++++++--- 11 files changed, 278 insertions(+), 80 deletions(-) diff --git a/remote/bitbucket/bitbucket.go b/remote/bitbucket/bitbucket.go index a272fa88..736f8882 100644 --- a/remote/bitbucket/bitbucket.go +++ b/remote/bitbucket/bitbucket.go @@ -183,7 +183,12 @@ func (c *config) Perm(u *model.User, owner, name string) (*model.Perm, error) { // File fetches the file from the Bitbucket repository and returns its contents. func (c *config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { - config, err := c.newClient(u).FindSource(r.Owner, r.Name, b.Commit, f) + return c.FileRef(u, r, b.Commit, f) +} + +// FileRef fetches the file from the Bitbucket repository and returns its contents. +func (c *config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + config, err := c.newClient(u).FindSource(r.Owner, r.Name, ref, f) if err != nil { return nil, err } diff --git a/remote/bitbucketserver/bitbucketserver.go b/remote/bitbucketserver/bitbucketserver.go index 6068e105..6e44baa3 100644 --- a/remote/bitbucketserver/bitbucketserver.go +++ b/remote/bitbucketserver/bitbucketserver.go @@ -9,14 +9,15 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/drone/drone/model" - "github.com/drone/drone/remote" - "github.com/drone/drone/remote/bitbucketserver/internal" - "github.com/mrjones/oauth" "io/ioutil" "net/http" "net/url" "strings" + + "github.com/drone/drone/model" + "github.com/drone/drone/remote" + "github.com/drone/drone/remote/bitbucketserver/internal" + "github.com/mrjones/oauth" ) const ( @@ -164,6 +165,12 @@ func (c *Config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return client.FindFileForRepo(r.Owner, r.Name, f, b.Ref) } +func (c *Config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + client := internal.NewClientWithToken(c.URL, c.Consumer, u.Token) + + return client.FindFileForRepo(r.Owner, r.Name, f, ref) +} + // Status is not supported by the bitbucketserver driver. func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { status := internal.BuildStatus{ diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go index 6b8be516..4554e3ff 100644 --- a/remote/gerrit/gerrit.go +++ b/remote/gerrit/gerrit.go @@ -94,6 +94,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return nil, nil } +// File is not supported by the Gerrit driver. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + return nil, nil +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/github/github.go b/remote/github/github.go index 7b38278b..ba3468f6 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -219,12 +219,17 @@ func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) { return convertPerm(repo), nil } -// File fetches the file from the Bitbucket repository and returns its contents. +// File fetches the file from the GitHub repository and returns its contents. func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { + return c.FileRef(u, r, b.Commit, f) +} + +// FileRef fetches the file from the GitHub repository and returns its contents. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { client := c.newClientToken(u.Token) opts := new(github.RepositoryContentGetOptions) - opts.Ref = b.Commit + opts.Ref = ref data, _, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts) if err != nil { return nil, err diff --git a/remote/gitlab/client/project.go b/remote/gitlab/client/project.go index 6140ab32..287655e2 100644 --- a/remote/gitlab/client/project.go +++ b/remote/gitlab/client/project.go @@ -7,11 +7,12 @@ import ( ) const ( - searchUrl = "/projects/search/:query" - projectsUrl = "/projects" - projectUrl = "/projects/:id" - repoUrlRawFile = "/projects/:id/repository/blobs/:sha" - commitStatusUrl = "/projects/:id/statuses/:sha" + searchUrl = "/projects/search/:query" + projectsUrl = "/projects" + projectUrl = "/projects/:id" + repoUrlRawFile = "/projects/:id/repository/blobs/:sha" + repoUrlRawFileRef = "/projects/:id/repository/files" + commitStatusUrl = "/projects/:id/statuses/:sha" ) // Get a list of all projects owned by the authenticated user. @@ -96,6 +97,23 @@ func (c *Client) RepoRawFile(id, sha, filepath string) ([]byte, error) { return contents, err } +func (c *Client) RepoRawFileRef(id, ref, filepath string) ([]byte, error) { + url, opaque := c.ResourceUrl( + repoUrlRawFileRef, + QMap{ + ":id": id, + }, + QMap{ + "filepath": filepath, + "ref": strings.TrimPrefix(ref, "refs/heads/"), + }, + ) + + contents, err := c.Do("GET", url, opaque, nil) + + return contents, err +} + // func (c *Client) SetStatus(id, sha, state, desc, ref, link string) error { url, opaque := c.ResourceUrl( diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index 728d8d8e..97e5eb77 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -321,6 +321,21 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f return out, err } +// FileRef fetches the file from the GitHub repository and returns its contents. +func (g *Gitlab) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + var client = NewClient(g.URL, u.Token, g.SkipVerify) + id, err := GetProjectId(g, client, r.Owner, r.Name) + if err != nil { + return nil, err + } + + out, err := client.RepoRawFileRef(id, ref, f) + if err != nil { + return nil, err + } + return out, err +} + // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. diff --git a/remote/gogs/gogs.go b/remote/gogs/gogs.go index e5e77df2..aa5b922e 100644 --- a/remote/gogs/gogs.go +++ b/remote/gogs/gogs.go @@ -190,6 +190,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } +// FileRef fetches the file from the Gogs repository and returns its contents. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + return c.newClientToken(u.Token).GetFile(r.Owner, r.Name, ref, f) +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/mock/remote.go b/remote/mock/remote.go index d4419471..0a6d4196 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -1,4 +1,4 @@ -package mock +package mocks import ( "net/http" @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/mock" ) -// This is an autogenerated mock type for the Remote type +// Remote is an autogenerated mock type for the Remote type type Remote struct { mock.Mock } @@ -84,6 +84,29 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ( return r0, r1 } +// FileRef provides a mock function with given fields: u, r, ref, f +func (_m *Remote) FileRef(u *model.User, r *model.Repo, ref string, f string) ([]byte, error) { + ret := _m.Called(u, r, ref, f) + + var r0 []byte + if rf, ok := ret.Get(0).(func(*model.User, *model.Repo, string, string) []byte); ok { + r0 = rf(u, r, ref, f) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.User, *model.Repo, string, string) error); ok { + r1 = rf(u, r, ref, f) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Hook provides a mock function with given fields: r func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) { ret := _m.Called(r) @@ -245,29 +268,6 @@ func (_m *Remote) Status(u *model.User, r *model.Repo, b *model.Build, link stri return r0 } -// Teams provides a mock function with given fields: u -func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) { - ret := _m.Called(u) - - var r0 []*model.Team - if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok { - r0 = rf(u) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Team) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.User) error); ok { - r1 = rf(u) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // TeamPerm provides a mock function with given fields: u, org func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) { ret := _m.Called(u, org) @@ -290,3 +290,26 @@ func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) { return r0, r1 } + +// Teams provides a mock function with given fields: u +func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) { + ret := _m.Called(u) + + var r0 []*model.Team + if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok { + r0 = rf(u) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Team) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.User) error); ok { + r1 = rf(u) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/remote/remote.go b/remote/remote.go index aac6aa53..ea7d0ec0 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -41,6 +41,10 @@ type Remote interface { // format. File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) + // FileRef fetches a file from the remote repository for the given ref + // and returns in string format. + FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) + // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. Status(u *model.User, r *model.Repo, b *model.Build, link string) error @@ -109,12 +113,12 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er // File fetches a file from the remote repository and returns in string format. func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { - for i:=0;i<5;i++ { + for i := 0; i < 5; i++ { out, err = FromContext(c).File(u, r, b, f) if err == nil { return } - time.Sleep(1*time.Second) + time.Sleep(1 * time.Second) } return } diff --git a/server/build.go b/server/build.go index 2f37396a..0028e95d 100644 --- a/server/build.go +++ b/server/build.go @@ -158,9 +158,10 @@ func DeleteBuild(c *gin.Context) { func PostApproval(c *gin.Context) { var ( - repo = session.Repo(c) - user = session.User(c) - num, _ = strconv.Atoi( + remote_ = remote.FromContext(c) + repo = session.Repo(c) + user = session.User(c) + num, _ = strconv.Atoi( c.Params.ByName("number"), ) ) @@ -178,16 +179,115 @@ func PostApproval(c *gin.Context) { build.Reviewed = time.Now().Unix() build.Reviewer = user.Login - if err := store.UpdateBuild(c, build); err != nil { - c.String(500, "error updating build. %s", err) + // + // + // This code is copied pasted until I have a chance + // to refactor into a proper function. Lots of changes + // and technical debt. No judgement please! + // + // + + // fetch the build file from the database + cfg := ToConfig(c) + raw, err := remote_.File(user, repo, build, cfg.Yaml) + if err != nil { + logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) + c.AbortWithError(404, err) return } - // - // TODO start build - // + netrc, err := remote_.Netrc(user, repo) + if err != nil { + c.String(500, "Failed to generate netrc file. %s", err) + return + } + + if uerr := store.UpdateBuild(c, build); err != nil { + c.String(500, "error updating build. %s", uerr) + return + } c.JSON(200, build) + + // get the previous build so that we can send + // on status change notifications + last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) + secs, err := store.GetMergedSecretList(c, repo) + if err != nil { + logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) + } + + defer func() { + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) + } + }() + + b := builder{ + Repo: repo, + Curr: build, + Last: last, + Netrc: netrc, + Secs: secs, + Link: httputil.GetURL(c.Request), + Yaml: string(raw), + } + items, err := b.Build() + if err != nil { + build.Status = model.StatusError + build.Started = time.Now().Unix() + build.Finished = build.Started + build.Error = err.Error() + store.UpdateBuild(c, build) + return + } + + for _, item := range items { + build.Jobs = append(build.Jobs, item.Job) + store.CreateJob(c, item.Job) + // TODO err + } + + // + // publish topic + // + message := pubsub.Message{ + Labels: map[string]string{ + "repo": repo.FullName, + "private": strconv.FormatBool(repo.IsPrivate), + }, + } + message.Data, _ = json.Marshal(model.Event{ + Type: model.Enqueued, + Repo: *repo, + Build: *build, + }) + // TODO remove global reference + config.pubsub.Publish(c, "topic/events", message) + // + // end publish topic + // + + for _, item := range items { + task := new(queue.Task) + task.ID = fmt.Sprint(item.Job.ID) + task.Labels = map[string]string{} + task.Labels["platform"] = item.Platform + for k, v := range item.Labels { + task.Labels[k] = v + } + + task.Data, _ = json.Marshal(rpc.Pipeline{ + ID: fmt.Sprint(item.Job.ID), + Config: item.Config, + Timeout: b.Repo.Timeout, + }) + + config.logger.Open(context.Background(), task.ID) + config.queue.Push(context.Background(), task) + } } func PostDecline(c *gin.Context) { @@ -219,20 +319,10 @@ func PostDecline(c *gin.Context) { return } - owner, err := store.GetUser(c, repo.UserID) - if err == nil { - if refresher, ok := remote_.(remote.Refresher); ok { - ok, _ := refresher.Refresh(user) - if ok { - store.UpdateUser(c, user) - } - } - - uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(owner, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) - } + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) } c.JSON(200, build) diff --git a/server/hook.go b/server/hook.go index 6ac9f6a4..e0d01c2a 100644 --- a/server/hook.go +++ b/server/hook.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "context" "encoding/json" "fmt" @@ -9,7 +10,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/square/go-jose" "github.com/Sirupsen/logrus" "github.com/drone/drone/model" @@ -139,10 +139,6 @@ func PostHook(c *gin.Context) { c.AbortWithError(404, err) return } - sec, err := remote_.File(user, repo, build, cfg.Shasum) - if err != nil { - logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err) - } netrc, err := remote_.Netrc(user, repo) if err != nil { @@ -159,26 +155,47 @@ func PostHook(c *gin.Context) { } } - signature, err := jose.ParseSigned(string(sec)) - if err != nil { - logrus.Debugf("cannot parse .drone.yml.sig file. %s", err) - } else if len(sec) == 0 { - logrus.Debugf("cannot parse .drone.yml.sig file. empty file") - } else { - build.Signed = true - output, verr := signature.Verify([]byte(repo.Hash)) - if verr != nil { - logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr) - } else if string(output) != string(raw) { - logrus.Debugf("cannot verify .drone.yml.sig file. no match") + // TODO default logic should avoid the approval if all + // secrets have skip-verify flag + + if build.Event == model.EventPull { + old, ferr := remote_.FileRef(user, repo, build.Ref, cfg.Yaml) + if ferr != nil { + build.Status = model.StatusBlocked + } else if bytes.Equal(old, raw) { + build.Status = model.StatusPending } else { - build.Verified = true + // this block is executed if the target yaml file + // does not match the base yaml. + + // TODO unfortunately we have no good way to get the + // sender repository permissions unless the user is + // a registered drone user. + sender, uerr := store.GetUserLogin(c, build.Sender) + if uerr != nil { + build.Status = model.StatusBlocked + } else { + if refresher, ok := remote_.(remote.Refresher); ok { + ok, _ := refresher.Refresh(sender) + if ok { + store.UpdateUser(c, sender) + } + } + // if the sender does not have push access to the + // repository the pull request should be blocked. + perm, perr := remote_.Perm(sender, repo.Owner, repo.Name) + if perr != nil || perm.Push == false { + build.Status = model.StatusBlocked + } + } } + } else { + build.Status = model.StatusPending } // update some build fields - build.Status = model.StatusPending build.RepoID = repo.ID + build.Verified = true if err := store.CreateBuild(c, build, build.Jobs...); err != nil { logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) @@ -188,6 +205,10 @@ func PostHook(c *gin.Context) { c.JSON(200, build) + if build.Status == model.StatusBlocked { + return + } + // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) From 45af1697b89d66eb6e72af5a079364cfc7fd1115 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 19:48:54 +0800 Subject: [PATCH 5/6] no need to trim file ref --- remote/gitlab/client/project.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote/gitlab/client/project.go b/remote/gitlab/client/project.go index 287655e2..f21f1a48 100644 --- a/remote/gitlab/client/project.go +++ b/remote/gitlab/client/project.go @@ -105,7 +105,7 @@ func (c *Client) RepoRawFileRef(id, ref, filepath string) ([]byte, error) { }, QMap{ "filepath": filepath, - "ref": strings.TrimPrefix(ref, "refs/heads/"), + "ref": ref, }, ) From bd25f57fa778893cb09a7b8ea4b4d744fc85251d Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 19:50:20 +0800 Subject: [PATCH 6/6] fix mock package name --- remote/mock/remote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/remote/mock/remote.go b/remote/mock/remote.go index 0a6d4196..f97a677f 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -1,4 +1,4 @@ -package mocks +package mock import ( "net/http"