darkkirb/harness-drone
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

compare yaml files

Browse source
This commit is contained in:
Brad Rydzewski 2017-03-18 19:25:53 +08:00
parent bfdd09c484
commit 130c623a35
11 changed files with 278 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
remote/bitbucket/bitbucket.go
View file

@ -183,7 +183,12 @@ func (c *config) Perm(u *model.User, owner, name string) (*model.Perm, error) {
// File fetches the file from the Bitbucket repository and returns its contents.
func (c *config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) {
config, err := c.newClient(u).FindSource(r.Owner, r.Name, b.Commit, f)
return c.FileRef(u, r, b.Commit, f)
}
// FileRef fetches the file from the Bitbucket repository and returns its contents.
func (c *config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
config, err := c.newClient(u).FindSource(r.Owner, r.Name, ref, f)
if err != nil {
return nil, err
}

15
remote/bitbucketserver/bitbucketserver.go
View file

@ -9,14 +9,15 @@ import (
"crypto/x509"
"encoding/pem"
"fmt"
"github.com/drone/drone/model"
"github.com/drone/drone/remote"
"github.com/drone/drone/remote/bitbucketserver/internal"
"github.com/mrjones/oauth"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/drone/drone/model"
"github.com/drone/drone/remote"
"github.com/drone/drone/remote/bitbucketserver/internal"
"github.com/mrjones/oauth"
)
const (
@ -164,6 +165,12 @@ func (c *Config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([
return client.FindFileForRepo(r.Owner, r.Name, f, b.Ref)
}
func (c *Config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
client := internal.NewClientWithToken(c.URL, c.Consumer, u.Token)
return client.FindFileForRepo(r.Owner, r.Name, f, ref)
}
// Status is not supported by the bitbucketserver driver.
func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error {
status := internal.BuildStatus{

5
remote/gerrit/gerrit.go
View file

@ -94,6 +94,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([
return nil, nil
}
// File is not supported by the Gerrit driver.
func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
return nil, nil
}
// Status is not supported by the Gogs driver.
func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error {
return nil

9
remote/github/github.go
View file

@ -219,12 +219,17 @@ func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) {
return convertPerm(repo), nil
}
// File fetches the file from the Bitbucket repository and returns its contents.
// File fetches the file from the GitHub repository and returns its contents.
func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) {
return c.FileRef(u, r, b.Commit, f)
}
// FileRef fetches the file from the GitHub repository and returns its contents.
func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
client := c.newClientToken(u.Token)
opts := new(github.RepositoryContentGetOptions)
opts.Ref = b.Commit
opts.Ref = ref
data, _, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts)
if err != nil {
return nil, err

28
remote/gitlab/client/project.go
View file

@ -7,11 +7,12 @@ import (
)
const (
searchUrl = "/projects/search/:query"
projectsUrl = "/projects"
projectUrl = "/projects/:id"
repoUrlRawFile = "/projects/:id/repository/blobs/:sha"
commitStatusUrl = "/projects/:id/statuses/:sha"
searchUrl = "/projects/search/:query"
projectsUrl = "/projects"
projectUrl = "/projects/:id"
repoUrlRawFile = "/projects/:id/repository/blobs/:sha"
repoUrlRawFileRef = "/projects/:id/repository/files"
commitStatusUrl = "/projects/:id/statuses/:sha"
)
// Get a list of all projects owned by the authenticated user.
@ -96,6 +97,23 @@ func (c *Client) RepoRawFile(id, sha, filepath string) ([]byte, error) {
return contents, err
}
func (c *Client) RepoRawFileRef(id, ref, filepath string) ([]byte, error) {
url, opaque := c.ResourceUrl(
repoUrlRawFileRef,
QMap{
":id": id,
},
QMap{
"filepath": filepath,
"ref": strings.TrimPrefix(ref, "refs/heads/"),
},
)
contents, err := c.Do("GET", url, opaque, nil)
return contents, err
}
//
func (c *Client) SetStatus(id, sha, state, desc, ref, link string) error {
url, opaque := c.ResourceUrl(

15
remote/gitlab/gitlab.go
View file

@ -321,6 +321,21 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f
return out, err
}
// FileRef fetches the file from the GitHub repository and returns its contents.
func (g *Gitlab) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
var client = NewClient(g.URL, u.Token, g.SkipVerify)
id, err := GetProjectId(g, client, r.Owner, r.Name)
if err != nil {
return nil, err
}
out, err := client.RepoRawFileRef(id, ref, f)
if err != nil {
return nil, err
}
return out, err
}
// NOTE Currently gitlab doesn't support status for commits and events,
// also if we want get MR status in gitlab we need implement a special plugin for gitlab,
// gitlab uses API to fetch build status on client side. But for now we skip this.

5
remote/gogs/gogs.go
View file

@ -190,6 +190,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([
return cfg, err
}
// FileRef fetches the file from the Gogs repository and returns its contents.
func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) {
return c.newClientToken(u.Token).GetFile(r.Owner, r.Name, ref, f)
}
// Status is not supported by the Gogs driver.
func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error {
return nil

73
remote/mock/remote.go
View file

@ -1,4 +1,4 @@
package mock
package mocks
import (
"net/http"
@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/mock"
)
// This is an autogenerated mock type for the Remote type
// Remote is an autogenerated mock type for the Remote type
type Remote struct {
mock.Mock
}
@ -84,6 +84,29 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) (
return r0, r1
}
// FileRef provides a mock function with given fields: u, r, ref, f
func (_m *Remote) FileRef(u *model.User, r *model.Repo, ref string, f string) ([]byte, error) {
ret := _m.Called(u, r, ref, f)
var r0 []byte
if rf, ok := ret.Get(0).(func(*model.User, *model.Repo, string, string) []byte); ok {
r0 = rf(u, r, ref, f)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*model.User, *model.Repo, string, string) error); ok {
r1 = rf(u, r, ref, f)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Hook provides a mock function with given fields: r
func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) {
ret := _m.Called(r)
@ -245,29 +268,6 @@ func (_m *Remote) Status(u *model.User, r *model.Repo, b *model.Build, link stri
return r0
}
// Teams provides a mock function with given fields: u
func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) {
ret := _m.Called(u)
var r0 []*model.Team
if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok {
r0 = rf(u)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.Team)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*model.User) error); ok {
r1 = rf(u)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// TeamPerm provides a mock function with given fields: u, org
func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) {
ret := _m.Called(u, org)
@ -290,3 +290,26 @@ func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return r0, r1
}
// Teams provides a mock function with given fields: u
func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) {
ret := _m.Called(u)
var r0 []*model.Team
if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok {
r0 = rf(u)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.Team)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*model.User) error); ok {
r1 = rf(u)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

8
remote/remote.go
View file

@ -41,6 +41,10 @@ type Remote interface {
// format.
File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error)
// FileRef fetches a file from the remote repository for the given ref
// and returns in string format.
FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error)
// Status sends the commit status to the remote system.
// An example would be the GitHub pull request status.
Status(u *model.User, r *model.Repo, b *model.Build, link string) error
@ -109,12 +113,12 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er
// File fetches a file from the remote repository and returns in string format.
func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) {
for i:=0;i<5;i++ {
for i := 0; i < 5; i++ {
out, err = FromContext(c).File(u, r, b, f)
if err == nil {
return
}
time.Sleep(1*time.Second)
time.Sleep(1 * time.Second)
}
return
}

134
server/build.go
View file

@ -158,9 +158,10 @@ func DeleteBuild(c *gin.Context) {
func PostApproval(c *gin.Context) {
var (
repo = session.Repo(c)
user = session.User(c)
num, _ = strconv.Atoi(
remote_ = remote.FromContext(c)
repo = session.Repo(c)
user = session.User(c)
num, _ = strconv.Atoi(
c.Params.ByName("number"),
)
)
@ -178,16 +179,115 @@ func PostApproval(c *gin.Context) {
build.Reviewed = time.Now().Unix()
build.Reviewer = user.Login
if err := store.UpdateBuild(c, build); err != nil {
c.String(500, "error updating build. %s", err)
//
//
// This code is copied pasted until I have a chance
// to refactor into a proper function. Lots of changes
// and technical debt. No judgement please!
//
//
// fetch the build file from the database
cfg := ToConfig(c)
raw, err := remote_.File(user, repo, build, cfg.Yaml)
if err != nil {
logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err)
c.AbortWithError(404, err)
return
}
//
// TODO start build
//
netrc, err := remote_.Netrc(user, repo)
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
return
}
if uerr := store.UpdateBuild(c, build); err != nil {
c.String(500, "error updating build. %s", uerr)
return
}
c.JSON(200, build)
// get the previous build so that we can send
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, err := store.GetMergedSecretList(c, repo)
if err != nil {
logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
}
defer func() {
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(user, repo, build, uri)
if err != nil {
logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number)
}
}()
b := builder{
Repo: repo,
Curr: build,
Last: last,
Netrc: netrc,
Secs: secs,
Link: httputil.GetURL(c.Request),
Yaml: string(raw),
}
items, err := b.Build()
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
build.Finished = build.Started
build.Error = err.Error()
store.UpdateBuild(c, build)
return
}
for _, item := range items {
build.Jobs = append(build.Jobs, item.Job)
store.CreateJob(c, item.Job)
// TODO err
}
//
// publish topic
//
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
message.Data, _ = json.Marshal(model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: *build,
})
// TODO remove global reference
config.pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Job.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Job.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
config.logger.Open(context.Background(), task.ID)
config.queue.Push(context.Background(), task)
}
}
func PostDecline(c *gin.Context) {
@ -219,20 +319,10 @@ func PostDecline(c *gin.Context) {
return
}
owner, err := store.GetUser(c, repo.UserID)
if err == nil {
if refresher, ok := remote_.(remote.Refresher); ok {
ok, _ := refresher.Refresh(user)
if ok {
store.UpdateUser(c, user)
}
}
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(owner, repo, build, uri)
if err != nil {
logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number)
}
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(user, repo, build, uri)
if err != nil {
logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number)
}
c.JSON(200, build)

59
server/hook.go
View file

@ -1,6 +1,7 @@
package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
@ -9,7 +10,6 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/square/go-jose"
"github.com/Sirupsen/logrus"
"github.com/drone/drone/model"
@ -139,10 +139,6 @@ func PostHook(c *gin.Context) {
c.AbortWithError(404, err)
return
}
sec, err := remote_.File(user, repo, build, cfg.Shasum)
if err != nil {
logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err)
}
netrc, err := remote_.Netrc(user, repo)
if err != nil {
@ -159,26 +155,47 @@ func PostHook(c *gin.Context) {
}
}
signature, err := jose.ParseSigned(string(sec))
if err != nil {
logrus.Debugf("cannot parse .drone.yml.sig file. %s", err)
} else if len(sec) == 0 {
logrus.Debugf("cannot parse .drone.yml.sig file. empty file")
} else {
build.Signed = true
output, verr := signature.Verify([]byte(repo.Hash))
if verr != nil {
logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr)
} else if string(output) != string(raw) {
logrus.Debugf("cannot verify .drone.yml.sig file. no match")
// TODO default logic should avoid the approval if all
// secrets have skip-verify flag
if build.Event == model.EventPull {
old, ferr := remote_.FileRef(user, repo, build.Ref, cfg.Yaml)
if ferr != nil {
build.Status = model.StatusBlocked
} else if bytes.Equal(old, raw) {
build.Status = model.StatusPending
} else {
build.Verified = true
// this block is executed if the target yaml file
// does not match the base yaml.
// TODO unfortunately we have no good way to get the
// sender repository permissions unless the user is
// a registered drone user.
sender, uerr := store.GetUserLogin(c, build.Sender)
if uerr != nil {
build.Status = model.StatusBlocked
} else {
if refresher, ok := remote_.(remote.Refresher); ok {
ok, _ := refresher.Refresh(sender)
if ok {
store.UpdateUser(c, sender)
}
}
// if the sender does not have push access to the
// repository the pull request should be blocked.
perm, perr := remote_.Perm(sender, repo.Owner, repo.Name)
if perr != nil || perm.Push == false {
build.Status = model.StatusBlocked
}
}
}
} else {
build.Status = model.StatusPending
}
// update some build fields
build.Status = model.StatusPending
build.RepoID = repo.ID
build.Verified = true
if err := store.CreateBuild(c, build, build.Jobs...); err != nil {
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
@ -188,6 +205,10 @@ func PostHook(c *gin.Context) {
c.JSON(200, build)
if build.Status == model.StatusBlocked {
return
}
// get the previous build so that we can send
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)