Merge pull request #1586 from bradrydzewski/master

fully functioning builds using 0.5 agents, secrets and yaml (behind feature flag)

api/build.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/drone/drone/bus"
 	"github.com/drone/drone/engine"
 	"github.com/drone/drone/queue"
 	"github.com/drone/drone/remote"
@@ -149,6 +150,12 @@ func DeleteBuild(c *gin.Context) {
 		c.AbortWithError(404, err)
 		return
 	}
+
+	if os.Getenv("CANARY") == "true" {
+		bus.Publish(c, bus.NewEvent(bus.Cancelled, repo, build, job))
+		return
+	}
+
 	node, err := store.GetNode(c, job.NodeID)
 	if err != nil {
 		c.AbortWithError(404, err)
@@ -280,7 +287,7 @@ func PostBuild(c *gin.Context) {
 	// get the previous build so that we can send
 	// on status change notifications
 	last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
-
+	secs, _ := store.GetSecretList(c, repo)
 
 	// IMPORTANT. PLEASE READ
 	//
@@ -289,6 +296,7 @@ func PostBuild(c *gin.Context) {
 	// enabled using with the environment variable CANARY=true
 
 	if os.Getenv("CANARY") == "true" {
+		bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
 		for _, job := range jobs {
 			queue.Publish(c, &queue.Work{
 				User:      user,
@@ -300,6 +308,7 @@ func PostBuild(c *gin.Context) {
 				Netrc:     netrc,
 				Yaml:      string(raw),
 				YamlEnc:   string(sec),
+				Secrets:   secs,
 				System: &model.System{
 					Link:      httputil.GetURL(c.Request),
 					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

api/queue.go

@@ -51,9 +51,7 @@ func Wait(c *gin.Context) {
 	for {
 		select {
 		case event := <-eventc:
-			if event.Job.ID == id &&
+			if event.Job.ID == id && event.Type == bus.Cancelled {
-				event.Job.Status != model.StatusPending &&
-				event.Job.Status != model.StatusRunning {
 				c.JSON(200, event.Job)
 				return
 			}
@@ -93,13 +91,18 @@ func Update(c *gin.Context) {
 	job.Status = work.Job.Status
 	job.ExitCode = work.Job.ExitCode
 
+	if build.Status == model.StatusPending {
+		build.Status = model.StatusRunning
+		store.UpdateBuild(c, build)
+	}
+
 	ok, err := store.UpdateBuildJob(c, build, job)
 	if err != nil {
 		c.String(500, "Unable to update job. %s", err)
 		return
 	}
 
-	if ok {
+	if ok && build.Status != model.StatusRunning {
 		// get the user because we transfer the user form the server to agent
 		// and back we lose the token which does not get serialized to json.
 		user, err := store.GetUser(c, work.User.ID)
@@ -107,10 +110,16 @@ func Update(c *gin.Context) {
 			c.String(500, "Unable to find user. %s", err)
 			return
 		}
-		bus.Publish(c, &bus.Event{})
 		remote.Status(c, user, work.Repo, build,
 			fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
 	}
+
+	if build.Status == model.StatusRunning {
+		bus.Publish(c, bus.NewEvent(bus.Started, work.Repo, build, job))
+	} else {
+		bus.Publish(c, bus.NewEvent(bus.Finished, work.Repo, build, job))
+	}
+
 	c.JSON(200, work)
 }
 

bus/types.go

@@ -30,3 +30,11 @@ func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
 		Job:   *j,
 	}
 }
+
+func NewBuildEvent(t EventType, r *model.Repo, b *model.Build) *Event {
+	return &Event{
+		Type:  t,
+		Repo:  *r,
+		Build: *b,
+	}
+}

client/client.go

@@ -70,16 +70,6 @@ func (c *client) Wait(id int64) *Wait {
 	return &Wait{id, c, ctx, cancel}
 }
 
-////////
-
-type CancelNotifier interface {
-	Canecel()
-	CancelNotify() bool
-	IsCancelled() bool
-}
-
-////////
-
 type Wait struct {
 	id     int64
 	client *client

drone/agent/exec.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/Sirupsen/logrus"
@@ -14,6 +16,8 @@ import (
 	"github.com/drone/drone/engine/runner"
 	engine "github.com/drone/drone/engine/runner/docker"
 	"github.com/drone/drone/model"
+	"github.com/drone/drone/queue"
+	"github.com/drone/drone/yaml/expander"
 
 	"github.com/samalba/dockerclient"
 	"golang.org/x/net/context"
@@ -40,57 +44,37 @@ func exec(client client.Client, docker dockerclient.Client) error {
 
 	prefix := fmt.Sprintf("drone_%s", uniuri.New())
 
+	envs := toEnv(w)
+	w.Yaml = expander.ExpandString(w.Yaml, envs)
+
+	w.Secrets = append(w.Secrets, &model.Secret{Name: "HEROKU_TOKEN", Value: "GODZILLA", Images: []string{"golang:1.4.2"}, Events: []string{w.Build.Event}})
+
 	trans := []compiler.Transform{
-		builtin.NewCloneOp("plugins/git:latest", true),
+		builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true),
 		builtin.NewCacheOp(
 			"plugins/cache:latest",
 			"/var/lib/drone/cache/"+w.Repo.FullName,
 			false,
 		),
+		builtin.NewSecretOp(w.Build.Event, w.Secrets),
 		builtin.NewNormalizeOp("plugins"),
 		builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
-		builtin.NewEnvOp(map[string]string{
-			"CI":                "drone",
-			"CI_REPO":           w.Repo.FullName,
-			"CI_REPO_OWNER":     w.Repo.Owner,
-			"CI_REPO_NAME":      w.Repo.Name,
-			"CI_REPO_LINK":      w.Repo.Link,
-			"CI_REPO_AVATAR":    w.Repo.Avatar,
-			"CI_REPO_BRANCH":    w.Repo.Branch,
-			"CI_REPO_PRIVATE":   fmt.Sprintf("%v", w.Repo.IsPrivate),
-			"CI_REMOTE_URL":     w.Repo.Clone,
-			"CI_COMMIT_SHA":     w.Build.Commit,
-			"CI_COMMIT_REF":     w.Build.Ref,
-			"CI_COMMIT_BRANCH":  w.Build.Branch,
-			"CI_COMMIT_LINK":    w.Build.Link,
-			"CI_COMMIT_MESSAGE": w.Build.Message,
-			"CI_AUTHOR":         w.Build.Author,
-			"CI_AUTHOR_EMAIL":   w.Build.Email,
-			"CI_AUTHOR_AVATAR":  w.Build.Avatar,
-			"CI_BUILD_NUMBER":   fmt.Sprintf("%v", w.Build.Number),
-			"CI_BUILD_EVENT":    w.Build.Event,
-			// "CI_NETRC_USERNAME":    w.Netrc.Login,
-			// "CI_NETRC_PASSWORD":    w.Netrc.Password,
-			// "CI_NETRC_MACHINE":     w.Netrc.Machine,
-			// "CI_PREV_BUILD_STATUS": w.BuildLast.Status,
-			// "CI_PREV_BUILD_NUMBER": fmt.Sprintf("%v", w.BuildLast.Number),
-			// "CI_PREV_COMMIT_SHA":   w.BuildLast.Commit,
-		}),
 		builtin.NewValidateOp(
 			w.Repo.IsTrusted,
 			[]string{"plugins/*"},
 		),
+		builtin.NewEnvOp(envs),
 		builtin.NewShellOp(builtin.Linux_adm64),
 		builtin.NewArgsOp(),
 		builtin.NewPodOp(prefix),
 		builtin.NewAliasOp(prefix),
 		builtin.NewPullOp(false),
 		builtin.NewFilterOp(
-			model.StatusSuccess, // w.BuildLast.Status,
+			model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here
 			w.Build.Branch,
 			w.Build.Event,
 			w.Build.Deploy,
-			map[string]string{},
+			w.Job.Environment,
 		),
 	}
 
@@ -171,7 +155,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
 	w.Job.Finished = time.Now().Unix()
 
 	switch w.Job.ExitCode {
-	case 128, 130:
+	case 128, 130, 137:
 		w.Job.Status = model.StatusKilled
 	case 0:
 		w.Job.Status = model.StatusSuccess
@@ -184,3 +168,68 @@ func exec(client client.Client, docker dockerclient.Client) error {
 
 	return client.Push(w)
 }
+
+func toEnv(w *queue.Work) map[string]string {
+	envs := map[string]string{
+		"CI":                   "drone",
+		"DRONE":                "true",
+		"DRONE_ARCH":           "linux_amd64",
+		"DRONE_REPO":           w.Repo.FullName,
+		"DRONE_REPO_SCM":       w.Repo.Kind,
+		"DRONE_REPO_OWNER":     w.Repo.Owner,
+		"DRONE_REPO_NAME":      w.Repo.Name,
+		"DRONE_REPO_LINK":      w.Repo.Link,
+		"DRONE_REPO_AVATAR":    w.Repo.Avatar,
+		"DRONE_REPO_BRANCH":    w.Repo.Branch,
+		"DRONE_REPO_PRIVATE":   fmt.Sprintf("%v", w.Repo.IsPrivate),
+		"DRONE_REPO_TRUSTED":   fmt.Sprintf("%v", w.Repo.IsTrusted),
+		"DRONE_REMOTE_URL":     w.Repo.Clone,
+		"DRONE_COMMIT_SHA":     w.Build.Commit,
+		"DRONE_COMMIT_REF":     w.Build.Ref,
+		"DRONE_COMMIT_BRANCH":  w.Build.Branch,
+		"DRONE_COMMIT_LINK":    w.Build.Link,
+		"DRONE_COMMIT_MESSAGE": w.Build.Message,
+		"DRONE_AUTHOR":         w.Build.Author,
+		"DRONE_AUTHOR_EMAIL":   w.Build.Email,
+		"DRONE_AUTHOR_AVATAR":  w.Build.Avatar,
+		"DRONE_BUILD_NUMBER":   fmt.Sprintf("%d", w.Build.Number),
+		"DRONE_BUILD_EVENT":    w.Build.Event,
+		"DRONE_BUILD_CREATED":  fmt.Sprintf("%d", w.Build.Created),
+		"DRONE_BUILD_STARTED":  fmt.Sprintf("%d", w.Build.Started),
+		"DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished),
+		"DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false),
+
+		// SHORTER ALIASES
+		"DRONE_BRANCH": w.Build.Branch,
+		"DRONE_COMMIT": w.Build.Commit,
+
+		// TODO(bradrydzewski) netrc should only be injected via secrets
+		// "DRONE_NETRC_USERNAME":    w.Netrc.Login,
+		// "DRONE_NETRC_PASSWORD":    w.Netrc.Password,
+		// "DRONE_NETRC_MACHINE":     w.Netrc.Machine,
+	}
+
+	if w.Build.Event == model.EventTag {
+		envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
+	}
+	if w.Build.Event == model.EventPull {
+		envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
+	}
+	if w.Build.Event == model.EventDeploy {
+		envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
+	}
+
+	if w.BuildLast != nil {
+		envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
+		envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
+		envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
+	}
+
+	// inject matrix values as environment variables
+	for key, val := range w.Job.Environment {
+		envs[key] = val
+	}
+	return envs
+}
+
+var pullRegexp = regexp.MustCompile("\\d+")

engine/compiler/builtin/secrets.go

@@ -0,0 +1,33 @@
+package builtin
+
+import (
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/model"
+)
+
+type secretOp struct {
+	visitor
+	event   string
+	secrets []*model.Secret
+}
+
+// NewSecretOp returns a transformer that configures plugin secrets.
+func NewSecretOp(event string, secrets []*model.Secret) Visitor {
+	return &secretOp{
+		event:   event,
+		secrets: secrets,
+	}
+}
+
+func (v *secretOp) VisitContainer(node *parse.ContainerNode) error {
+	for _, secret := range v.secrets {
+		if !secret.Match(node.Container.Image, v.event) {
+			continue
+		}
+		if node.Container.Environment == nil {
+			node.Container.Environment = map[string]string{}
+		}
+		node.Container.Environment[secret.Name] = secret.Value
+	}
+	return nil
+}

engine/compiler/builtin/validate.go

@@ -103,3 +103,18 @@ func (v *validateOp) validateConfig(node *parse.ContainerNode) error {
 	}
 	return nil
 }
+
+// validate the environment configuration and return an error if
+// an attempt is made to override system environment variables.
+// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error {
+// 	for key := range node.Container.Environment {
+// 		upper := strings.ToUpper(key)
+// 		switch {
+// 		case strings.HasPrefix(upper, "DRONE_"):
+// 			return fmt.Errorf("Cannot set or override DRONE_ environment variables")
+// 		case strings.HasPrefix(upper, "PLUGIN_"):
+// 			return fmt.Errorf("Cannot set or override PLUGIN_ environment variables")
+// 		}
+// 	}
+// 	return nil
+// }

model/repo.go

@@ -1,7 +1,5 @@
 package model
 
-import "strconv"
-
 type RepoLite struct {
 	Owner    string `json:"owner"`
 	Name     string `json:"name"`
@@ -33,17 +31,3 @@ type Repo struct {
 	AllowTag    bool   `json:"allow_tags"        meddler:"repo_allow_tags"`
 	Hash        string `json:"-"                 meddler:"repo_hash"`
 }
-
-// ToEnv returns environment variable valus for the repository.
-func (r *Repo) ToEnv(to map[string]string) {
-	to["CI_VCS"] = r.Kind
-	to["CI_REPO"] = r.FullName
-	to["CI_REPO_OWNER"] = r.Owner
-	to["CI_REPO_NAME"] = r.Name
-	to["CI_REPO_LINK"] = r.Link
-	to["CI_REPO_AVATAR"] = r.Avatar
-	to["CI_REPO_BRANCH"] = r.Branch
-	to["CI_REPO_PRIVATE"] = strconv.FormatBool(r.IsPrivate)
-	to["CI_REPO_TRUSTED"] = strconv.FormatBool(r.IsTrusted)
-	to["CI_REMOTE_URL"] = r.Clone
-}

model/secret.go

@@ -22,6 +22,32 @@ type Secret struct {
 	Events []string `json:"event,omitempty" meddler:"secret_events,json"`
 }
 
+// Match returns true if an image and event match the restricted list.
+func (s *Secret) Match(image, event string) bool {
+	return s.MatchImage(image) && s.MatchEvent(event)
+}
+
+// MatchImage returns true if an image matches the restricted list.
+func (s *Secret) MatchImage(want string) bool {
+	for _, got := range s.Images {
+		if want == got {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchEvent returns true if an event matches the restricted list.
+func (s *Secret) MatchEvent(want string) bool {
+	for _, got := range s.Events {
+		if want == got {
+			return true
+		}
+	}
+	return false
+}
+
+// Validate validates the required fields and formats.
 func (s *Secret) Validate() error {
 	return nil
 }

model/secret_test.go

@@ -0,0 +1,40 @@
+package model
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestSecret(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("Secret", func() {
+
+		g.It("should match image", func() {
+			secret := Secret{}
+			secret.Images = []string{"golang"}
+			g.Assert(secret.MatchImage("golang")).IsTrue()
+		})
+		g.It("should match event", func() {
+			secret := Secret{}
+			secret.Events = []string{"pull_request"}
+			g.Assert(secret.MatchEvent("pull_request")).IsTrue()
+		})
+		g.It("should not match image", func() {
+			secret := Secret{}
+			secret.Images = []string{"golang"}
+			g.Assert(secret.MatchImage("node")).IsFalse()
+		})
+		g.It("should not match event", func() {
+			secret := Secret{}
+			secret.Events = []string{"pull_request"}
+			g.Assert(secret.MatchEvent("push")).IsFalse()
+		})
+		g.It("should pass validation")
+		g.Describe("should fail validation", func() {
+			g.It("when no image")
+			g.It("when no event")
+		})
+	})
+}

queue/types.go

@@ -5,14 +5,15 @@ import "github.com/drone/drone/model"
 // Work represents an item for work to be
 // processed by a worker.
 type Work struct {
-	Yaml      string        `json:"config"`
+	Yaml      string          `json:"config"`
-	YamlEnc   string        `json:"secret"`
+	YamlEnc   string          `json:"secret"`
-	Repo      *model.Repo   `json:"repo"`
+	Repo      *model.Repo     `json:"repo"`
-	Build     *model.Build  `json:"build"`
+	Build     *model.Build    `json:"build"`
-	BuildLast *model.Build  `json:"build_last"`
+	BuildLast *model.Build    `json:"build_last"`
-	Job       *model.Job    `json:"job"`
+	Job       *model.Job      `json:"job"`
-	Netrc     *model.Netrc  `json:"netrc"`
+	Netrc     *model.Netrc    `json:"netrc"`
-	Keys      *model.Key    `json:"keys"`
+	Keys      *model.Key      `json:"keys"`
-	System    *model.System `json:"system"`
+	System    *model.System   `json:"system"`
-	User      *model.User   `json:"user"`
+	Secrets   []*model.Secret `json:"secret"`
+	User      *model.User     `json:"user"`
 }

store/store.go

@@ -279,6 +279,13 @@ func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool
 	if err := UpdateJob(c, job); err != nil {
 		return false, err
 	}
+
+	// if the job is running or started we don't need to update the build
+	// status since.
+	if job.Status == model.StatusRunning || job.Status == model.StatusPending {
+		return false, nil
+	}
+
 	jobs, err := GetJobList(c, build)
 	if err != nil {
 		return false, err

web/hook.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/drone/drone/bus"
 	"github.com/drone/drone/engine"
 	"github.com/drone/drone/model"
 	"github.com/drone/drone/queue"
@@ -204,6 +205,7 @@ func PostHook(c *gin.Context) {
 	// get the previous build so that we can send
 	// on status change notifications
 	last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
+	secs, _ := store.GetSecretList(c, repo)
 
 	// IMPORTANT. PLEASE READ
 	//
@@ -212,6 +214,7 @@ func PostHook(c *gin.Context) {
 	// enabled using with the environment variable CANARY=true
 
 	if os.Getenv("CANARY") == "true" {
+		bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
 		for _, job := range jobs {
 			queue.Publish(c, &queue.Work{
 				User:      user,
@@ -223,6 +226,7 @@ func PostHook(c *gin.Context) {
 				Netrc:     netrc,
 				Yaml:      string(raw),
 				YamlEnc:   string(sec),
+				Secrets:   secs,
 				System: &model.System{
 					Link:      httputil.GetURL(c.Request),
 					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

web/stream2.go

@@ -46,6 +46,8 @@ func GetRepoEvents2(c *gin.Context) {
 				return false
 			}
 
+			// TODO(bradrydzewski) This is a super hacky workaround until we improve
+			// the actual bus. Having a per-call database event is just plain stupid.
 			if event.Repo.FullName == repo.FullName {
 
 				var payload = struct {