Merge pull request #1591 from bradrydzewski/master

use new 0.5 .drone.sig signature file

api/build.go

@@ -18,6 +18,7 @@ import (
 	"github.com/drone/drone/shared/httputil"
 	"github.com/drone/drone/store"
 	"github.com/gin-gonic/gin"
+	"github.com/square/go-jose"
 
 	"github.com/drone/drone/model"
 	"github.com/drone/drone/router/middleware/session"
@@ -33,6 +34,9 @@ func init() {
 		droneYml = ".drone.yml"
 	}
 	droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
+	if os.Getenv("CANARY") == "true" {
+		droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
+	}
 }
 
 func GetBuilds(c *gin.Context) {
@@ -296,25 +300,33 @@ func PostBuild(c *gin.Context) {
 	// enabled using with the environment variable CANARY=true
 
 	if os.Getenv("CANARY") == "true" {
+
+		var signed bool
+		var verified bool
+
+		signature, err := jose.ParseSigned(string(sec))
+		if err == nil && len(sec) != 0 {
+			signed = true
+			output, err := signature.Verify(repo.Hash)
+			if err == nil && string(output) == string(raw) {
+				verified = true
+			}
+		}
+
 		bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
 		for _, job := range jobs {
 			queue.Publish(c, &queue.Work{
+				Signed:    signed,
+				Verified:  verified,
 				User:      user,
 				Repo:      repo,
 				Build:     build,
 				BuildLast: last,
 				Job:       job,
-				Keys:      key,
 				Netrc:     netrc,
 				Yaml:      string(raw),
-				YamlEnc:   string(sec),
 				Secrets:   secs,
-				System: &model.System{
-					Link:      httputil.GetURL(c.Request),
-					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
-					Globals:   strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
-					Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
-				},
+				System:    &model.System{Link: httputil.GetURL(c.Request)},
 			})
 		}
 		return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW

client/client.go

@@ -18,7 +18,7 @@ import (
 )
 
 const (
-	pathPull   = "%s/api/queue/pull"
+	pathPull   = "%s/api/queue/pull/%s/%s"
 	pathWait   = "%s/api/queue/wait/%d"
 	pathStream = "%s/api/queue/stream/%d"
 	pathPush   = "%s/api/queue/status/%d"
@@ -43,9 +43,9 @@ func NewClientToken(uri, token string) Client {
 }
 
 // Pull pulls work from the server queue.
-func (c *client) Pull() (*queue.Work, error) {
+func (c *client) Pull(os, arch string) (*queue.Work, error) {
 	out := new(queue.Work)
-	uri := fmt.Sprintf(pathPull, c.base)
+	uri := fmt.Sprintf(pathPull, c.base, os, arch)
 	err := c.post(uri, nil, out)
 	return out, err
 }

client/interface.go

@@ -9,7 +9,7 @@ import (
 // Client is used to communicate with a Drone server.
 type Client interface {
 	// Pull pulls work from the server queue.
-	Pull() (*queue.Work, error)
+	Pull(os, arch string) (*queue.Work, error)
 
 	// Push pushes an update to the server.
 	Push(*queue.Work) error

drone/agent/agent.go

@@ -40,6 +40,18 @@ var AgentCmd = cli.Command{
 			Usage:  "limit number of running docker processes",
 			Value:  2,
 		},
+		cli.StringFlag{
+			EnvVar: "DOCKER_OS",
+			Name:   "docker-os",
+			Usage:  "docker operating system",
+			Value:  "linux",
+		},
+		cli.StringFlag{
+			EnvVar: "DOCKER_ARCH",
+			Name:   "docker-arch",
+			Usage:  "docker architecture system",
+			Value:  "amd64",
+		},
 		cli.StringFlag{
 			EnvVar: "DRONE_SERVER",
 			Name:   "drone-server",
@@ -68,16 +80,40 @@ var AgentCmd = cli.Command{
 			Usage:  "start the agent with experimental features",
 		},
 		cli.StringSliceFlag{
-			EnvVar: "DRONE_NETRC_PLUGIN",
+			EnvVar: "DRONE_PLUGIN_NETRC",
 			Name:   "netrc-plugin",
 			Usage:  "plugins that receive the netrc file",
 			Value:  &cli.StringSlice{"git", "hg"},
 		},
 		cli.StringSliceFlag{
-			EnvVar: "DRONE_PRIVILEGED_PLUGIN",
-			Name:   "privileged-plugin",
+			EnvVar: "DRONE_PLUGIN_PRIVILEGED",
+			Name:   "privileged",
 			Usage:  "plugins that require privileged mode",
-			Value:  &cli.StringSlice{"docker", "gcr", "ecr"},
+			Value: &cli.StringSlice{
+				"plugins/docker",
+				"plugins/docker:*",
+				"plguins/gcr",
+				"plguins/gcr:*",
+				"plugins/ecr",
+				"plugins/ecr:*",
+			},
+		},
+		cli.BoolFlag{
+			EnvVar: "DRONE_PLUGIN_PULL",
+			Name:   "pull",
+			Usage:  "always pull latest plugin images",
+		},
+		cli.StringFlag{
+			EnvVar: "DRONE_PLUGIN_NAMESPACE",
+			Name:   "namespace",
+			Value:  "plugins",
+			Usage:  "default plugin image namespace",
+		},
+		cli.StringSliceFlag{
+			EnvVar: "DRONE_PLUGIN_WHITELIST",
+			Name:   "whitelist",
+			Usage:  "plugins that are permitted to run on the host",
+			Value:  &cli.StringSlice{"plugins/*"},
 		},
 	},
 }
@@ -109,10 +145,21 @@ func start(c *cli.Context) {
 	for i := 0; i < c.Int("docker-max-procs"); i++ {
 		wg.Add(1)
 		go func() {
+			r := pipeline{
+				drone:  client,
+				docker: docker,
+				config: config{
+					whitelist:  c.StringSlice("whitelist"),
+					namespace:  c.String("namespace"),
+					privileged: c.StringSlice("privileged"),
+					netrc:      c.StringSlice("netrc-plugin"),
+					pull:       c.Bool("pull"),
+				},
+			}
 			for {
-				if err := recoverExec(client, docker); err != nil {
+				if err := r.run(); err != nil {
 					dur := c.Duration("backoff")
-					logrus.Debugf("Attempting to reconnect in %v", dur)
+					logrus.Warnf("Attempting to reconnect in %v", dur)
 					time.Sleep(dur)
 				}
 			}

drone/agent/exec.go

@@ -14,7 +14,7 @@ import (
 	"github.com/drone/drone/engine/compiler"
 	"github.com/drone/drone/engine/compiler/builtin"
 	"github.com/drone/drone/engine/runner"
-	engine "github.com/drone/drone/engine/runner/docker"
+	"github.com/drone/drone/engine/runner/docker"
 	"github.com/drone/drone/model"
 	"github.com/drone/drone/queue"
 	"github.com/drone/drone/yaml/expander"
@@ -23,15 +23,23 @@ import (
 	"golang.org/x/net/context"
 )
 
-func recoverExec(client client.Client, docker dockerclient.Client) error {
-	defer func() {
-		recover()
-	}()
-	return exec(client, docker)
+type config struct {
+	platform   string
+	namespace  string
+	whitelist  []string
+	privileged []string
+	netrc      []string
+	pull       bool
 }
 
-func exec(client client.Client, docker dockerclient.Client) error {
-	w, err := client.Pull()
+type pipeline struct {
+	drone  client.Client
+	docker dockerclient.Client
+	config config
+}
+
+func (r *pipeline) run() error {
+	w, err := r.drone.Pull("linux", "amd64")
 	if err != nil {
 		return err
 	}
@@ -46,23 +54,34 @@ func exec(client client.Client, docker dockerclient.Client) error {
 
 	envs := toEnv(w)
 	w.Yaml = expander.ExpandString(w.Yaml, envs)
+	if w.Verified {
+
+	}
+	if w.Signed {
+
+	}
 
 	// inject the netrc file into the clone plugin if the repositroy is
 	// private and requires authentication.
+	var secrets []*model.Secret
+	if w.Verified {
+		secrets = append(secrets, w.Secrets...)
+	}
+
 	if w.Repo.IsPrivate {
-		w.Secrets = append(w.Secrets, &model.Secret{
+		secrets = append(secrets, &model.Secret{
 			Name:   "DRONE_NETRC_USERNAME",
 			Value:  w.Netrc.Login,
 			Images: []string{"git", "hg"}, // TODO(bradrydzewski) use the command line parameters here
 			Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag},
 		})
-		w.Secrets = append(w.Secrets, &model.Secret{
+		secrets = append(secrets, &model.Secret{
 			Name:   "DRONE_NETRC_PASSWORD",
 			Value:  w.Netrc.Password,
-			Images: []string{w.Repo.Kind},
+			Images: []string{"git", "hg"},
 			Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag},
 		})
-		w.Secrets = append(w.Secrets, &model.Secret{
+		secrets = append(secrets, &model.Secret{
 			Name:   "DRONE_NETRC_MACHINE",
 			Value:  w.Netrc.Machine,
 			Images: []string{"git", "hg"},
@@ -71,25 +90,26 @@ func exec(client client.Client, docker dockerclient.Client) error {
 	}
 
 	trans := []compiler.Transform{
-		builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true),
+		builtin.NewCloneOp("plugins/git:latest", true),
 		builtin.NewCacheOp(
 			"plugins/cache:latest",
 			"/var/lib/drone/cache/"+w.Repo.FullName,
 			false,
 		),
-		builtin.NewSecretOp(w.Build.Event, w.Secrets),
-		builtin.NewNormalizeOp("plugins"),
-		builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
+		builtin.NewSecretOp(w.Build.Event, secrets),
+		builtin.NewNormalizeOp(r.config.namespace),
+		builtin.NewWorkspaceOp("/drone", "/drone/src/github.com/"+w.Repo.FullName),
 		builtin.NewValidateOp(
 			w.Repo.IsTrusted,
-			[]string{"plugins/*"},
+			r.config.whitelist,
 		),
 		builtin.NewEnvOp(envs),
 		builtin.NewShellOp(builtin.Linux_adm64),
 		builtin.NewArgsOp(),
+		builtin.NewEscalateOp(r.config.privileged),
 		builtin.NewPodOp(prefix),
 		builtin.NewAliasOp(prefix),
-		builtin.NewPullOp(false),
+		builtin.NewPullOp(r.config.pull),
 		builtin.NewFilterOp(
 			model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here
 			w.Build.Branch,
@@ -109,14 +129,14 @@ func exec(client client.Client, docker dockerclient.Client) error {
 		return err
 	}
 
-	if err := client.Push(w); err != nil {
+	if err := r.drone.Push(w); err != nil {
 		logrus.Errorf("Error persisting update %s/%s#%d.%d. %s",
 			w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
 		return err
 	}
 
 	conf := runner.Config{
-		Engine: engine.New(docker),
+		Engine: docker.New(r.docker),
 	}
 
 	ctx := context.TODO()
@@ -126,7 +146,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
 	run.Run()
 	defer cancel()
 
-	wait := client.Wait(w.Job.ID)
+	wait := r.drone.Wait(w.Job.ID)
 	if err != nil {
 		return err
 	}
@@ -142,7 +162,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
 
 	rc, wc := io.Pipe()
 	go func() {
-		err := client.Stream(w.Job.ID, rc)
+		err := r.drone.Stream(w.Job.ID, rc)
 		if err != nil && err != io.ErrClosedPipe {
 			logrus.Errorf("Error streaming build logs. %s", err)
 		}
@@ -187,7 +207,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
 	logrus.Infof("Finished build %s/%s#%d.%d",
 		w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
 
-	return client.Push(w)
+	return r.drone.Push(w)
 }
 
 func toEnv(w *queue.Work) map[string]string {
@@ -218,7 +238,8 @@ func toEnv(w *queue.Work) map[string]string {
 		"DRONE_BUILD_CREATED":  fmt.Sprintf("%d", w.Build.Created),
 		"DRONE_BUILD_STARTED":  fmt.Sprintf("%d", w.Build.Started),
 		"DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished),
-		"DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false),
+		"DRONE_YAML_VERIFIED":  fmt.Sprintf("%v", w.Verified),
+		"DRONE_YAML_SIGNED":    fmt.Sprintf("%v", w.Signed),
 
 		// SHORTER ALIASES
 		"DRONE_BRANCH": w.Build.Branch,

engine/compiler/builtin/escalate.go

@@ -0,0 +1,30 @@
+package builtin
+
+import (
+	"path/filepath"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type escalateOp struct {
+	visitor
+	plugins []string
+}
+
+// NewEscalateOp returns a transformer that configures plugins to automatically
+// execute in privileged mode. This is intended for plugins running dind.
+func NewEscalateOp(plugins []string) Visitor {
+	return &escalateOp{
+		plugins: plugins,
+	}
+}
+
+func (v *escalateOp) VisitContainer(node *parse.ContainerNode) error {
+	for _, pattern := range v.plugins {
+		ok, _ := filepath.Match(pattern, node.Container.Image)
+		if ok {
+			node.Container.Privileged = true
+		}
+	}
+	return nil
+}

engine/compiler/builtin/escalate_test.go

@@ -0,0 +1,54 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_escalate(t *testing.T) {
+	root := parse.NewRootNode()
+
+	g := goblin.Goblin(t)
+	g.Describe("privileged transform", func() {
+
+		g.It("should handle matches", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{Image: "plugins/docker"}
+			op := NewEscalateOp([]string{"plugins/docker"})
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Privileged).IsTrue()
+		})
+
+		g.It("should handle glob matches", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{Image: "plugins/docker"}
+			op := NewEscalateOp([]string{"plugins/*"})
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Privileged).IsTrue()
+		})
+
+		g.It("should handle non matches", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{Image: "plugins/git"}
+			op := NewEscalateOp([]string{"plugins/docker"})
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Privileged).IsFalse()
+		})
+
+		g.It("should handle non glob matches", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{Image: "plugins/docker:develop"}
+			op := NewEscalateOp([]string{"plugins/docker"})
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Privileged).IsFalse()
+		})
+	})
+}

engine/compiler/builtin/normalize.go

@@ -43,6 +43,9 @@ func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) {
 	if strings.Contains(node.Container.Image, "/") {
 		return
 	}
+	if strings.Contains(node.Container.Image, "_") {
+		node.Container.Image = strings.Replace(node.Container.Image, "_", "-", -1)
+	}
 	node.Container.Image = filepath.Join(v.namespace, node.Container.Image)
 }
 

engine/compiler/builtin/normalize_test.go

@@ -56,6 +56,15 @@ func Test_normalize(t *testing.T) {
 				g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest")
 			})
 
+			g.It("should replace underscores with dashes", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "gh_pages"}
+				op := NewNormalizeOp("plugins")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("plugins/gh-pages:latest")
+			})
+
 			g.It("should ignore shell or service types", func() {
 				c := root.NewShellNode()
 				c.Container = runner.Container{Image: "golang"}

queue/types.go

@@ -5,6 +5,8 @@ import "github.com/drone/drone/model"
 // Work represents an item for work to be
 // processed by a worker.
 type Work struct {
+	Signed    bool            `json:"signed"`
+	Verified  bool            `json:"verified"`
 	Yaml      string          `json:"config"`
 	YamlEnc   string          `json:"secret"`
 	Repo      *model.Repo     `json:"repo"`

router/middleware/agent.go

@@ -0,0 +1,45 @@
+package middleware
+
+import (
+	"github.com/drone/drone/shared/token"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/gin-gonic/gin"
+	"github.com/ianschenck/envflag"
+)
+
+var (
+	secret = envflag.String("AGENT_SECRET", "", "")
+	noauth = envflag.Bool("AGENT_NO_AUTH", false, "")
+)
+
+// Agent is a middleware function that initializes the authorization middleware
+// for agents to connect to the queue.
+func AgentMust() gin.HandlerFunc {
+
+	if *secret == "" {
+		logrus.Fatalf("please provide the agent secret to authenticate agent requests")
+	}
+
+	t := token.New(token.AgentToken, "")
+	s, err := t.Sign(*secret)
+	if err != nil {
+		logrus.Fatalf("invalid agent secret. %s", err)
+	}
+
+	logrus.Infof("using agent secret %s", *secret)
+	logrus.Warnf("agents can connect with token %s", s)
+
+	return func(c *gin.Context) {
+		parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) {
+			return *secret, nil
+		})
+		if err != nil {
+			c.AbortWithError(403, err)
+		} else if parsed.Kind != token.AgentToken {
+			c.AbortWithStatus(403)
+		} else {
+			c.Next()
+		}
+	}
+}

router/middleware/session/user.go

@@ -70,15 +70,14 @@ func MustAdmin() gin.HandlerFunc {
 		user := User(c)
 		switch {
 		case user == nil:
-			c.AbortWithStatus(http.StatusUnauthorized)
-			// c.HTML(http.StatusUnauthorized, "401.html", gin.H{})
+			c.String(401, "User not authorized")
+			c.Abort()
 		case user.Admin == false:
-			c.AbortWithStatus(http.StatusForbidden)
-			// c.HTML(http.StatusForbidden, "401.html", gin.H{})
+			c.String(413, "User not authorized")
+			c.Abort()
 		default:
 			c.Next()
 		}
-
 	}
 }
 
@@ -87,11 +86,10 @@ func MustUser() gin.HandlerFunc {
 		user := User(c)
 		switch {
 		case user == nil:
-			c.AbortWithStatus(http.StatusUnauthorized)
-			// c.HTML(http.StatusUnauthorized, "401.html", gin.H{})
+			c.String(401, "User not authorized")
+			c.Abort()
 		default:
 			c.Next()
 		}
-
 	}
 }

router/router.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"github.com/drone/drone/api"
+	"github.com/drone/drone/router/middleware"
 	"github.com/drone/drone/router/middleware/header"
 	"github.com/drone/drone/router/middleware/session"
 	"github.com/drone/drone/router/middleware/token"
@@ -16,7 +17,7 @@ import (
 	"github.com/drone/drone/web"
 )
 
-func Load(middleware ...gin.HandlerFunc) http.Handler {
+func Load(middlewares ...gin.HandlerFunc) http.Handler {
 	e := gin.New()
 	e.Use(gin.Recovery())
 
@@ -26,7 +27,7 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
 	e.Use(header.NoCache)
 	e.Use(header.Options)
 	e.Use(header.Secure)
-	e.Use(middleware...)
+	e.Use(middlewares...)
 	e.Use(session.SetUser())
 	e.Use(token.Refresh)
 
@@ -163,7 +164,9 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
 
 	queue := e.Group("/api/queue")
 	{
+		queue.Use(middleware.AgentMust())
 		queue.POST("/pull", api.Pull)
+		queue.POST("/pull/:os/:arch", api.Pull)
 		queue.POST("/wait/:id", api.Wait)
 		queue.POST("/stream/:id", api.Stream)
 		queue.POST("/status/:id", api.Update)

shared/token/token.go

@@ -10,10 +10,11 @@ import (
 type SecretFunc func(*Token) (string, error)
 
 const (
-	UserToken = "user"
-	SessToken = "sess"
-	HookToken = "hook"
-	CsrfToken = "csrf"
+	UserToken  = "user"
+	SessToken  = "sess"
+	HookToken  = "hook"
+	CsrfToken  = "csrf"
+	AgentToken = "agent"
 )
 
 // Default algorithm used to sign JWT tokens.

web/hook.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/gin-gonic/gin"
+	"github.com/square/go-jose"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/drone/drone/bus"
@@ -31,6 +32,9 @@ func init() {
 		droneYml = ".drone.yml"
 	}
 	droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
+	if os.Getenv("CANARY") == "true" {
+		droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
+	}
 }
 
 var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
@@ -214,25 +218,33 @@ func PostHook(c *gin.Context) {
 	// enabled using with the environment variable CANARY=true
 
 	if os.Getenv("CANARY") == "true" {
+
+		var signed bool
+		var verified bool
+
+		signature, err := jose.ParseSigned(string(sec))
+		if err == nil && len(sec) != 0 {
+			signed = true
+			output, err := signature.Verify(repo.Hash)
+			if err == nil && string(output) == string(raw) {
+				verified = true
+			}
+		}
+
 		bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
 		for _, job := range jobs {
 			queue.Publish(c, &queue.Work{
+				Signed:    signed,
+				Verified:  verified,
 				User:      user,
 				Repo:      repo,
 				Build:     build,
 				BuildLast: last,
 				Job:       job,
-				Keys:      key,
 				Netrc:     netrc,
 				Yaml:      string(raw),
-				YamlEnc:   string(sec),
 				Secrets:   secs,
-				System: &model.System{
-					Link:      httputil.GetURL(c.Request),
-					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
-					Globals:   strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
-					Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
-				},
+				System:    &model.System{Link: httputil.GetURL(c.Request)},
 			})
 		}
 		return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW