darkkirb/harness-drone
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #1586 from bradrydzewski/master

Browse source 
fully functioning builds using 0.5 agents, secrets and yaml (behind feature flag)
This commit is contained in:
Brad Rydzewski 2016-04-21 00:30:41 -07:00
commit a0f8457e87
14 changed files with 250 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
api/build.go
View file

@ -11,6 +11,7 @@ import (
"time"
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/engine"
"github.com/drone/drone/queue"
"github.com/drone/drone/remote"
@ -149,6 +150,12 @@ func DeleteBuild(c *gin.Context) {
c.AbortWithError(404, err)
return
}
if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewEvent(bus.Cancelled, repo, build, job))
return
}
node, err := store.GetNode(c, job.NodeID)
if err != nil {
c.AbortWithError(404, err)
@ -280,7 +287,7 @@ func PostBuild(c *gin.Context) {
// get the previous build so that we can send
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, _ := store.GetSecretList(c, repo)
// IMPORTANT. PLEASE READ
//
@ -289,6 +296,7 @@ func PostBuild(c *gin.Context) {
// enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs {
queue.Publish(c, &queue.Work{
User: user,
@ -300,6 +308,7 @@ func PostBuild(c *gin.Context) {
Netrc: netrc,
Yaml: string(raw),
YamlEnc: string(sec),
Secrets: secs,
System: &model.System{
Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

19
api/queue.go
View file

@ -51,9 +51,7 @@ func Wait(c *gin.Context) {
for {
select {
case event := <-eventc:
if event.Job.ID == id &&
event.Job.Status != model.StatusPending &&
event.Job.Status != model.StatusRunning {
if event.Job.ID == id && event.Type == bus.Cancelled {
c.JSON(200, event.Job)
return
}
@ -93,13 +91,18 @@ func Update(c *gin.Context) {
job.Status = work.Job.Status
job.ExitCode = work.Job.ExitCode
if build.Status == model.StatusPending {
build.Status = model.StatusRunning
store.UpdateBuild(c, build)
}
ok, err := store.UpdateBuildJob(c, build, job)
if err != nil {
c.String(500, "Unable to update job. %s", err)
return
}
if ok {
if ok && build.Status != model.StatusRunning {
// get the user because we transfer the user form the server to agent
// and back we lose the token which does not get serialized to json.
user, err := store.GetUser(c, work.User.ID)
@ -107,10 +110,16 @@ func Update(c *gin.Context) {
c.String(500, "Unable to find user. %s", err)
return
}
bus.Publish(c, &bus.Event{})
remote.Status(c, user, work.Repo, build,
fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
}
if build.Status == model.StatusRunning {
bus.Publish(c, bus.NewEvent(bus.Started, work.Repo, build, job))
} else {
bus.Publish(c, bus.NewEvent(bus.Finished, work.Repo, build, job))
}
c.JSON(200, work)
}

8
bus/types.go
View file

@ -30,3 +30,11 @@ func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
Job: *j,
}
}
func NewBuildEvent(t EventType, r *model.Repo, b *model.Build) *Event {
return &Event{
Type: t,
Repo: *r,
Build: *b,
}
}

10
client/client.go
View file

@ -70,16 +70,6 @@ func (c *client) Wait(id int64) *Wait {
return &Wait{id, c, ctx, cancel}
}
////////
type CancelNotifier interface {
Canecel()
CancelNotify() bool
IsCancelled() bool
}
////////
type Wait struct {
id int64
client *client

111
drone/agent/exec.go
View file

@ -4,6 +4,8 @@ import (
"encoding/json"
"fmt"
"io"
"regexp"
"strings"
"time"
"github.com/Sirupsen/logrus"
@ -14,6 +16,8 @@ import (
"github.com/drone/drone/engine/runner"
engine "github.com/drone/drone/engine/runner/docker"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/yaml/expander"
"github.com/samalba/dockerclient"
"golang.org/x/net/context"
@ -40,57 +44,37 @@ func exec(client client.Client, docker dockerclient.Client) error {
prefix := fmt.Sprintf("drone_%s", uniuri.New())
envs := toEnv(w)
w.Yaml = expander.ExpandString(w.Yaml, envs)
w.Secrets = append(w.Secrets, &model.Secret{Name: "HEROKU_TOKEN", Value: "GODZILLA", Images: []string{"golang:1.4.2"}, Events: []string{w.Build.Event}})
trans := []compiler.Transform{
builtin.NewCloneOp("plugins/git:latest", true),
builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true),
builtin.NewCacheOp(
"plugins/cache:latest",
"/var/lib/drone/cache/"+w.Repo.FullName,
false,
),
builtin.NewSecretOp(w.Build.Event, w.Secrets),
builtin.NewNormalizeOp("plugins"),
builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
builtin.NewEnvOp(map[string]string{
"CI": "drone",
"CI_REPO": w.Repo.FullName,
"CI_REPO_OWNER": w.Repo.Owner,
"CI_REPO_NAME": w.Repo.Name,
"CI_REPO_LINK": w.Repo.Link,
"CI_REPO_AVATAR": w.Repo.Avatar,
"CI_REPO_BRANCH": w.Repo.Branch,
"CI_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate),
"CI_REMOTE_URL": w.Repo.Clone,
"CI_COMMIT_SHA": w.Build.Commit,
"CI_COMMIT_REF": w.Build.Ref,
"CI_COMMIT_BRANCH": w.Build.Branch,
"CI_COMMIT_LINK": w.Build.Link,
"CI_COMMIT_MESSAGE": w.Build.Message,
"CI_AUTHOR": w.Build.Author,
"CI_AUTHOR_EMAIL": w.Build.Email,
"CI_AUTHOR_AVATAR": w.Build.Avatar,
"CI_BUILD_NUMBER": fmt.Sprintf("%v", w.Build.Number),
"CI_BUILD_EVENT": w.Build.Event,
// "CI_NETRC_USERNAME": w.Netrc.Login,
// "CI_NETRC_PASSWORD": w.Netrc.Password,
// "CI_NETRC_MACHINE": w.Netrc.Machine,
// "CI_PREV_BUILD_STATUS": w.BuildLast.Status,
// "CI_PREV_BUILD_NUMBER": fmt.Sprintf("%v", w.BuildLast.Number),
// "CI_PREV_COMMIT_SHA": w.BuildLast.Commit,
}),
builtin.NewValidateOp(
w.Repo.IsTrusted,
[]string{"plugins/*"},
),
builtin.NewEnvOp(envs),
builtin.NewShellOp(builtin.Linux_adm64),
builtin.NewArgsOp(),
builtin.NewPodOp(prefix),
builtin.NewAliasOp(prefix),
builtin.NewPullOp(false),
builtin.NewFilterOp(
model.StatusSuccess, // w.BuildLast.Status,
model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here
w.Build.Branch,
w.Build.Event,
w.Build.Deploy,
map[string]string{},
w.Job.Environment,
),
}
@ -171,7 +155,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
w.Job.Finished = time.Now().Unix()
switch w.Job.ExitCode {
case 128, 130:
case 128, 130, 137:
w.Job.Status = model.StatusKilled
case 0:
w.Job.Status = model.StatusSuccess
@ -184,3 +168,68 @@ func exec(client client.Client, docker dockerclient.Client) error {
return client.Push(w)
}
func toEnv(w *queue.Work) map[string]string {
envs := map[string]string{
"CI": "drone",
"DRONE": "true",
"DRONE_ARCH": "linux_amd64",
"DRONE_REPO": w.Repo.FullName,
"DRONE_REPO_SCM": w.Repo.Kind,
"DRONE_REPO_OWNER": w.Repo.Owner,
"DRONE_REPO_NAME": w.Repo.Name,
"DRONE_REPO_LINK": w.Repo.Link,
"DRONE_REPO_AVATAR": w.Repo.Avatar,
"DRONE_REPO_BRANCH": w.Repo.Branch,
"DRONE_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate),
"DRONE_REPO_TRUSTED": fmt.Sprintf("%v", w.Repo.IsTrusted),
"DRONE_REMOTE_URL": w.Repo.Clone,
"DRONE_COMMIT_SHA": w.Build.Commit,
"DRONE_COMMIT_REF": w.Build.Ref,
"DRONE_COMMIT_BRANCH": w.Build.Branch,
"DRONE_COMMIT_LINK": w.Build.Link,
"DRONE_COMMIT_MESSAGE": w.Build.Message,
"DRONE_AUTHOR": w.Build.Author,
"DRONE_AUTHOR_EMAIL": w.Build.Email,
"DRONE_AUTHOR_AVATAR": w.Build.Avatar,
"DRONE_BUILD_NUMBER": fmt.Sprintf("%d", w.Build.Number),
"DRONE_BUILD_EVENT": w.Build.Event,
"DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created),
"DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started),
"DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished),
"DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false),
// SHORTER ALIASES
"DRONE_BRANCH": w.Build.Branch,
"DRONE_COMMIT": w.Build.Commit,
// TODO(bradrydzewski) netrc should only be injected via secrets
// "DRONE_NETRC_USERNAME": w.Netrc.Login,
// "DRONE_NETRC_PASSWORD": w.Netrc.Password,
// "DRONE_NETRC_MACHINE": w.Netrc.Machine,
}
if w.Build.Event == model.EventTag {
envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
}
if w.Build.Event == model.EventPull {
envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
}
if w.Build.Event == model.EventDeploy {
envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
}
if w.BuildLast != nil {
envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
}
// inject matrix values as environment variables
for key, val := range w.Job.Environment {
envs[key] = val
}
return envs
}
var pullRegexp = regexp.MustCompile("\\d+")

33
engine/compiler/builtin/secrets.go Normal file
View file

@ -0,0 +1,33 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/model"
)
type secretOp struct {
visitor
event string
secrets []*model.Secret
}
// NewSecretOp returns a transformer that configures plugin secrets.
func NewSecretOp(event string, secrets []*model.Secret) Visitor {
return &secretOp{
event: event,
secrets: secrets,
}
}
func (v *secretOp) VisitContainer(node *parse.ContainerNode) error {
for _, secret := range v.secrets {
if !secret.Match(node.Container.Image, v.event) {
continue
}
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
node.Container.Environment[secret.Name] = secret.Value
}
return nil
}

15
engine/compiler/builtin/validate.go
View file

@ -103,3 +103,18 @@ func (v *validateOp) validateConfig(node *parse.ContainerNode) error {
}
return nil
}
// validate the environment configuration and return an error if
// an attempt is made to override system environment variables.
// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error {
// for key := range node.Container.Environment {
// upper := strings.ToUpper(key)
// switch {
// case strings.HasPrefix(upper, "DRONE_"):
// return fmt.Errorf("Cannot set or override DRONE_ environment variables")
// case strings.HasPrefix(upper, "PLUGIN_"):
// return fmt.Errorf("Cannot set or override PLUGIN_ environment variables")
// }
// }
// return nil
// }

16
model/repo.go
View file

@ -1,7 +1,5 @@
package model
import "strconv"
type RepoLite struct {
Owner string `json:"owner"`
Name string `json:"name"`
@ -33,17 +31,3 @@ type Repo struct {
AllowTag bool `json:"allow_tags" meddler:"repo_allow_tags"`
Hash string `json:"-" meddler:"repo_hash"`
}
// ToEnv returns environment variable valus for the repository.
func (r *Repo) ToEnv(to map[string]string) {
to["CI_VCS"] = r.Kind
to["CI_REPO"] = r.FullName
to["CI_REPO_OWNER"] = r.Owner
to["CI_REPO_NAME"] = r.Name
to["CI_REPO_LINK"] = r.Link
to["CI_REPO_AVATAR"] = r.Avatar
to["CI_REPO_BRANCH"] = r.Branch
to["CI_REPO_PRIVATE"] = strconv.FormatBool(r.IsPrivate)
to["CI_REPO_TRUSTED"] = strconv.FormatBool(r.IsTrusted)
to["CI_REMOTE_URL"] = r.Clone
}

26
model/secret.go
View file

@ -22,6 +22,32 @@ type Secret struct {
Events []string `json:"event,omitempty" meddler:"secret_events,json"`
}
// Match returns true if an image and event match the restricted list.
func (s *Secret) Match(image, event string) bool {
return s.MatchImage(image) && s.MatchEvent(event)
}
// MatchImage returns true if an image matches the restricted list.
func (s *Secret) MatchImage(want string) bool {
for _, got := range s.Images {
if want == got {
return true
}
}
return false
}
// MatchEvent returns true if an event matches the restricted list.
func (s *Secret) MatchEvent(want string) bool {
for _, got := range s.Events {
if want == got {
return true
}
}
return false
}
// Validate validates the required fields and formats.
func (s *Secret) Validate() error {
return nil
}

40
model/secret_test.go Normal file
View file

@ -0,0 +1,40 @@
package model
import (
"testing"
"github.com/franela/goblin"
)
func TestSecret(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Secret", func() {
g.It("should match image", func() {
secret := Secret{}
secret.Images = []string{"golang"}
g.Assert(secret.MatchImage("golang")).IsTrue()
})
g.It("should match event", func() {
secret := Secret{}
secret.Events = []string{"pull_request"}
g.Assert(secret.MatchEvent("pull_request")).IsTrue()
})
g.It("should not match image", func() {
secret := Secret{}
secret.Images = []string{"golang"}
g.Assert(secret.MatchImage("node")).IsFalse()
})
g.It("should not match event", func() {
secret := Secret{}
secret.Events = []string{"pull_request"}
g.Assert(secret.MatchEvent("push")).IsFalse()
})
g.It("should pass validation")
g.Describe("should fail validation", func() {
g.It("when no image")
g.It("when no event")
})
})
}

1
queue/types.go
View file

@ -14,5 +14,6 @@ type Work struct {
Netrc *model.Netrc `json:"netrc"`
Keys *model.Key `json:"keys"`
System *model.System `json:"system"`
Secrets []*model.Secret `json:"secret"`
User *model.User `json:"user"`
}

7
store/store.go
View file

@ -279,6 +279,13 @@ func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool
if err := UpdateJob(c, job); err != nil {
return false, err
}
// if the job is running or started we don't need to update the build
// status since.
if job.Status == model.StatusRunning || job.Status == model.StatusPending {
return false, nil
}
jobs, err := GetJobList(c, build)
if err != nil {
return false, err

4
web/hook.go
View file

@ -10,6 +10,7 @@ import (
"github.com/gin-gonic/gin"
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/engine"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
@ -204,6 +205,7 @@ func PostHook(c *gin.Context) {
// get the previous build so that we can send
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, _ := store.GetSecretList(c, repo)
// IMPORTANT. PLEASE READ
//
@ -212,6 +214,7 @@ func PostHook(c *gin.Context) {
// enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs {
queue.Publish(c, &queue.Work{
User: user,
@ -223,6 +226,7 @@ func PostHook(c *gin.Context) {
Netrc: netrc,
Yaml: string(raw),
YamlEnc: string(sec),
Secrets: secs,
System: &model.System{
Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

2
web/stream2.go
View file

@ -46,6 +46,8 @@ func GetRepoEvents2(c *gin.Context) {
return false
}
// TODO(bradrydzewski) This is a super hacky workaround until we improve
// the actual bus. Having a per-call database event is just plain stupid.
if event.Repo.FullName == repo.FullName {
var payload = struct {