simplify build engine for reliability

This commit is contained in:
Brad Rydzewski 2016-05-09 22:57:57 -07:00
parent b0879fe47e
commit 3d05659134
38 changed files with 1371 additions and 280 deletions

1
build/convert.go Normal file
View file

@ -0,0 +1 @@
package build

View file

@ -1,4 +1,4 @@
package interpreter
package build
import (
"errors"

View file

@ -1,4 +1,4 @@
package interpreter
package build
import (
"testing"

View file

@ -1,4 +1,4 @@
package interpreter
package build
import "fmt"

View file

@ -1,4 +1,4 @@
package interpreter
package build
import (
"sync"

View file

@ -1,4 +1,4 @@
package interpreter
package build
import (
"bufio"
@ -7,8 +7,8 @@ import (
"strings"
"time"
"github.com/drone/drone/build/internal"
"github.com/drone/drone/yaml"
"github.com/drone/drone/yaml/interpreter/internal"
"github.com/samalba/dockerclient"
)

View file

@ -1,4 +1,4 @@
package interpreter
package build
import (
"fmt"

View file

@ -1 +1,453 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"net/url"
"os"
"os/signal"
"path/filepath"
"strings"
"time"
"github.com/drone/drone/build"
"github.com/drone/drone/model"
"github.com/drone/drone/yaml"
"github.com/drone/drone/yaml/expander"
"github.com/drone/drone/yaml/transform"
"github.com/codegangsta/cli"
"github.com/samalba/dockerclient"
)
var execCmd = cli.Command{
Name: "exec",
Usage: "execute a local build",
Action: func(c *cli.Context) {
if err := exec(c); err != nil {
log.Fatalln(err)
}
},
Flags: []cli.Flag{
cli.BoolTFlag{
Name: "local",
Usage: "build from local directory",
EnvVar: "DRONE_LOCAL",
},
cli.StringSliceFlag{
Name: "plugin",
Usage: "plugin steps to enable",
EnvVar: "DRONE_PLUGIN_ENABLE",
},
cli.StringSliceFlag{
Name: "secret",
Usage: "build secrets in KEY=VALUE format",
EnvVar: "DRONE_SECRET",
},
cli.StringSliceFlag{
Name: "matrix",
Usage: "build matrix in KEY=VALUE format",
EnvVar: "DRONE_MATRIX",
},
cli.DurationFlag{
Name: "timeout",
Usage: "build timeout for inactivity",
Value: time.Hour,
EnvVar: "DRONE_TIMEOUT",
},
cli.DurationFlag{
Name: "duration",
Usage: "build duration",
Value: time.Hour,
EnvVar: "DRONE_DURATION",
},
cli.BoolFlag{
EnvVar: "DRONE_PLUGIN_PULL",
Name: "pull",
Usage: "always pull latest plugin images",
},
cli.StringFlag{
EnvVar: "DRONE_PLUGIN_NAMESPACE",
Name: "namespace",
Value: "plugins",
Usage: "default plugin image namespace",
},
cli.StringSliceFlag{
EnvVar: "DRONE_PLUGIN_PRIVILEGED",
Name: "privileged",
Usage: "plugins that require privileged mode",
Value: &cli.StringSlice{
"plugins/docker",
"plugins/docker:*",
"plguins/gcr",
"plguins/gcr:*",
"plugins/ecr",
"plugins/ecr:*",
},
},
// Docker daemon flags
cli.StringFlag{
EnvVar: "DOCKER_HOST",
Name: "docker-host",
Usage: "docker deamon address",
Value: "unix:///var/run/docker.sock",
},
cli.BoolFlag{
EnvVar: "DOCKER_TLS_VERIFY",
Name: "docker-tls-verify",
Usage: "docker daemon supports tlsverify",
},
cli.StringFlag{
EnvVar: "DOCKER_CERT_PATH",
Name: "docker-cert-path",
Usage: "docker certificate directory",
Value: "",
},
//
// Please note the below flags are mirrored in the plugin starter kit and
// should be kept synchronized.
// https://github.com/drone/drone-plugin-starter
//
cli.StringFlag{
Name: "repo.fullname",
Usage: "repository full name",
EnvVar: "DRONE_REPO",
},
cli.StringFlag{
Name: "repo.owner",
Usage: "repository owner",
EnvVar: "DRONE_REPO_OWNER",
},
cli.StringFlag{
Name: "repo.name",
Usage: "repository name",
EnvVar: "DRONE_REPO_NAME",
},
cli.StringFlag{
Name: "repo.type",
Value: "git",
Usage: "repository type",
EnvVar: "DRONE_REPO_SCM",
},
cli.StringFlag{
Name: "repo.link",
Usage: "repository link",
EnvVar: "DRONE_REPO_LINK",
},
cli.StringFlag{
Name: "repo.avatar",
Usage: "repository avatar",
EnvVar: "DRONE_REPO_AVATAR",
},
cli.StringFlag{
Name: "repo.branch",
Usage: "repository default branch",
EnvVar: "DRONE_REPO_BRANCH",
},
cli.BoolFlag{
Name: "repo.private",
Usage: "repository is private",
EnvVar: "DRONE_REPO_PRIVATE",
},
cli.BoolFlag{
Name: "repo.trusted",
Usage: "repository is trusted",
EnvVar: "DRONE_REPO_TRUSTED",
},
cli.StringFlag{
Name: "remote.url",
Usage: "git remote url",
EnvVar: "DRONE_REMOTE_URL",
},
cli.StringFlag{
Name: "commit.sha",
Usage: "git commit sha",
EnvVar: "DRONE_COMMIT_SHA",
},
cli.StringFlag{
Name: "commit.ref",
Value: "refs/heads/master",
Usage: "git commit ref",
EnvVar: "DRONE_COMMIT_REF",
},
cli.StringFlag{
Name: "commit.branch",
Value: "master",
Usage: "git commit branch",
EnvVar: "DRONE_COMMIT_BRANCH",
},
cli.StringFlag{
Name: "commit.message",
Usage: "git commit message",
EnvVar: "DRONE_COMMIT_MESSAGE",
},
cli.StringFlag{
Name: "commit.link",
Usage: "git commit link",
EnvVar: "DRONE_COMMIT_LINK",
},
cli.StringFlag{
Name: "commit.author.name",
Usage: "git author name",
EnvVar: "DRONE_COMMIT_AUTHOR",
},
cli.StringFlag{
Name: "commit.author.email",
Usage: "git author email",
EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL",
},
cli.StringFlag{
Name: "commit.author.avatar",
Usage: "git author avatar",
EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR",
},
cli.StringFlag{
Name: "build.event",
Value: "push",
Usage: "build event",
EnvVar: "DRONE_BUILD_EVENT",
},
cli.IntFlag{
Name: "build.number",
Usage: "build number",
EnvVar: "DRONE_BUILD_NUMBER",
},
cli.IntFlag{
Name: "build.created",
Usage: "build created",
EnvVar: "DRONE_BUILD_CREATED",
},
cli.IntFlag{
Name: "build.started",
Usage: "build started",
EnvVar: "DRONE_BUILD_STARTED",
},
cli.IntFlag{
Name: "build.finished",
Usage: "build finished",
EnvVar: "DRONE_BUILD_FINISHED",
},
cli.StringFlag{
Name: "build.status",
Usage: "build status",
Value: "success",
EnvVar: "DRONE_BUILD_STATUS",
},
cli.StringFlag{
Name: "build.link",
Usage: "build link",
EnvVar: "DRONE_BUILD_LINK",
},
cli.StringFlag{
Name: "build.deploy",
Usage: "build deployment target",
EnvVar: "DRONE_DEPLOY_TO",
},
cli.BoolFlag{
Name: "yaml.verified",
Usage: "build yaml is verified",
EnvVar: "DRONE_YAML_VERIFIED",
},
cli.BoolFlag{
Name: "yaml.signed",
Usage: "build yaml is signed",
EnvVar: "DRONE_YAML_SIGNED",
},
cli.IntFlag{
Name: "prev.build.number",
Usage: "previous build number",
EnvVar: "DRONE_PREV_BUILD_NUMBER",
},
cli.StringFlag{
Name: "prev.build.status",
Usage: "previous build status",
EnvVar: "DRONE_PREV_BUILD_STATUS",
},
cli.StringFlag{
Name: "prev.commit.sha",
Usage: "previous build sha",
EnvVar: "DRONE_PREV_COMMIT_SHA",
},
cli.StringFlag{
Name: "netrc.username",
Usage: "previous build sha",
EnvVar: "DRONE_NETRC_USERNAME",
},
cli.StringFlag{
Name: "netrc.password",
Usage: "previous build sha",
EnvVar: "DRONE_NETRC_PASSWORD",
},
cli.StringFlag{
Name: "netrc.machine",
Usage: "previous build sha",
EnvVar: "DRONE_NETRC_MACHINE",
},
},
}
func exec(c *cli.Context) error {
// get environment variables from flags
var envs = map[string]string{}
for _, flag := range c.Command.Flags {
switch f := flag.(type) {
case cli.StringFlag:
envs[f.EnvVar] = c.String(f.Name)
case cli.IntFlag:
envs[f.EnvVar] = c.String(f.Name)
case cli.BoolFlag:
envs[f.EnvVar] = c.String(f.Name)
}
}
// get matrix variales from flags
for _, s := range c.StringSlice("matrix") {
parts := strings.SplitN(s, "=", 2)
if len(parts) != 2 {
continue
}
k := parts[0]
v := parts[1]
envs[k] = v
}
// get secret variales from flags
for _, s := range c.StringSlice("secret") {
parts := strings.SplitN(s, "=", 2)
if len(parts) != 2 {
continue
}
k := parts[0]
v := parts[1]
envs[k] = v
}
// builtin.NewFilterOp(
// c.String("prev.build.status"),
// c.String("commit.branch"),
// c.String("build.event"),
// c.String("build.deploy"),
// envs,
// ),
// }
sigterm := make(chan os.Signal, 1)
signal.Notify(sigterm, os.Interrupt)
path := c.Args().First()
if path == "" {
path = ".drone.yml"
}
path, _ = filepath.Abs(path)
dir := filepath.Dir(path)
file, err := ioutil.ReadFile(path)
if err != nil {
return err
}
// unmarshal the Yaml file with expanded environment variables.
conf, err := yaml.Parse(expander.Expand(file, envs))
if err != nil {
return err
}
tls, err := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path"))
if err == nil {
tls.InsecureSkipVerify = c.Bool("docker-tls-verify")
}
client, err := dockerclient.NewDockerClient(c.String("docker-host"), tls)
if err != nil {
return err
}
src := "src"
if url, _ := url.Parse(c.String("repo.link")); url != nil {
src = filepath.Join(src, url.Host, url.Path)
}
transform.Clone(conf, "git")
transform.Environ(conf, envs)
transform.DefaultFilter(conf)
transform.PluginDisable(conf, c.StringSlice("plugin"))
// transform.Secret(conf, secrets)
transform.Identifier(conf)
transform.WorkspaceTransform(conf, "/drone", src)
if err := transform.Check(conf, c.Bool("repo.trusted")); err != nil {
return err
}
transform.CommandTransform(conf)
transform.ImagePull(conf, c.Bool("pull"))
transform.ImageTag(conf)
transform.ImageName(conf)
transform.ImageNamespace(conf, c.String("namespace"))
transform.ImageEscalate(conf, c.StringSlice("privileged"))
if c.BoolT("local") {
transform.ImageVolume(conf, []string{dir + ":" + conf.Workspace.Path})
}
transform.PluginParams(conf)
transform.Pod(conf)
timeout := time.After(c.Duration("duration"))
// load the Yaml into the pipeline
pipeline := build.Load(conf, client)
defer pipeline.Teardown()
// setup the build environment
err = pipeline.Setup()
if err != nil {
return err
}
for {
select {
case <-pipeline.Done():
return pipeline.Err()
case <-sigterm:
pipeline.Stop()
return fmt.Errorf("interrupt received, build cancelled")
case <-timeout:
pipeline.Stop()
return fmt.Errorf("maximum time limit exceeded, build cancelled")
case <-time.After(c.Duration("timeout")):
pipeline.Stop()
return fmt.Errorf("terminal inactive for %v, build cancelled", c.Duration("timeout"))
case <-pipeline.Next():
// TODO(bradrydzewski) this entire block of code should probably get
// encapsulated in the pipeline.
status := model.StatusSuccess
if pipeline.Err() != nil {
status = model.StatusFailure
}
if !pipeline.Head().Constraints.Match(
"linux/amd64",
c.String("build.deploy"),
c.String("build.event"),
c.String("commit.branch"),
status, envs) {
pipeline.Skip()
} else {
pipeline.Exec()
pipeline.Head().Environment["DRONE_STATUS"] = status
}
case line := <-pipeline.Pipe():
println(line.String())
}
}
}

View file

@ -34,6 +34,7 @@ func main() {
agent.AgentCmd,
buildCmd,
deployCmd,
execCmd,
infoCmd,
secretCmd,
serverCmd,

View file

@ -152,7 +152,7 @@ func PostHook(c *gin.Context) {
// verify the branches can be built vs skipped
branches := yaml.ParseBranch(raw)
if !branches.Matches(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
if !branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
c.String(200, "Branch does not match restrictions defined in yaml")
return
}

View file

@ -1,79 +1,18 @@
package yaml
import (
"path/filepath"
"github.com/drone/drone/yaml/types"
"gopkg.in/yaml.v2"
)
type Branch struct {
Include []string
Exclude []string
}
import "gopkg.in/yaml.v2"
// ParseBranch parses the branch section of the Yaml document.
func ParseBranch(in []byte) *Branch {
return parseBranch(in)
func ParseBranch(in []byte) Constraint {
out := struct {
Constraint Constraint `yaml:"branches"`
}{}
yaml.Unmarshal(in, &out)
return out.Constraint
}
// ParseBranchString parses the branch section of the Yaml document.
func ParseBranchString(in string) *Branch {
func ParseBranchString(in string) Constraint {
return ParseBranch([]byte(in))
}
// Matches returns true if the branch matches the include patterns and does not
// match any of the exclude patterns.
func (b *Branch) Matches(branch string) bool {
// when no includes or excludes automatically match
if len(b.Include) == 0 && len(b.Exclude) == 0 {
return true
}
// exclusions are processed first. So we can include everything and then
// selectively exclude certain sub-patterns.
for _, pattern := range b.Exclude {
if pattern == branch {
return false
}
if ok, _ := filepath.Match(pattern, branch); ok {
return false
}
}
for _, pattern := range b.Include {
if pattern == branch {
return true
}
if ok, _ := filepath.Match(pattern, branch); ok {
return true
}
}
return false
}
func parseBranch(in []byte) *Branch {
out1 := struct {
Branch struct {
Include types.StringOrSlice `yaml:"include"`
Exclude types.StringOrSlice `yaml:"exclude"`
} `yaml:"branches"`
}{}
out2 := struct {
Include types.StringOrSlice `yaml:"branches"`
}{}
yaml.Unmarshal(in, &out1)
yaml.Unmarshal(in, &out2)
return &Branch{
Exclude: out1.Branch.Exclude.Slice(),
Include: append(
out1.Branch.Include.Slice(),
out2.Include.Slice()...,
),
}
}

View file

@ -13,62 +13,32 @@ func TestBranch(t *testing.T) {
g.It("Should parse and match emtpy", func() {
branch := ParseBranchString("")
g.Assert(branch.Matches("master")).IsTrue()
g.Assert(branch.Match("master")).IsTrue()
})
g.It("Should parse and match", func() {
branch := ParseBranchString("branches: { include: [ master, develop ] }")
g.Assert(branch.Matches("master")).IsTrue()
g.Assert(branch.Match("master")).IsTrue()
})
g.It("Should parse and match shortand", func() {
branch := ParseBranchString("branches: [ master, develop ]")
g.Assert(branch.Matches("master")).IsTrue()
g.Assert(branch.Match("master")).IsTrue()
})
g.It("Should parse and match shortand string", func() {
branch := ParseBranchString("branches: master")
g.Assert(branch.Matches("master")).IsTrue()
g.Assert(branch.Match("master")).IsTrue()
})
g.It("Should parse and match exclude", func() {
branch := ParseBranchString("branches: { exclude: [ master, develop ] }")
g.Assert(branch.Matches("master")).IsFalse()
g.Assert(branch.Match("master")).IsFalse()
})
g.It("Should parse and match exclude shorthand", func() {
branch := ParseBranchString("branches: { exclude: master }")
g.Assert(branch.Matches("master")).IsFalse()
})
g.It("Should match include", func() {
b := Branch{}
b.Include = []string{"master"}
g.Assert(b.Matches("master")).IsTrue()
})
g.It("Should match include pattern", func() {
b := Branch{}
b.Include = []string{"feature/*"}
g.Assert(b.Matches("feature/foo")).IsTrue()
})
g.It("Should fail to match include pattern", func() {
b := Branch{}
b.Include = []string{"feature/*"}
g.Assert(b.Matches("master")).IsFalse()
})
g.It("Should match exclude", func() {
b := Branch{}
b.Exclude = []string{"master"}
g.Assert(b.Matches("master")).IsFalse()
})
g.It("Should match exclude pattern", func() {
b := Branch{}
b.Exclude = []string{"feature/*"}
g.Assert(b.Matches("feature/foo")).IsFalse()
g.Assert(branch.Match("master")).IsFalse()
})
})
}

View file

@ -1,49 +1,152 @@
package yaml
import (
"path/filepath"
"github.com/drone/drone/yaml/types"
)
// Constraints define constraints for container execution.
type Constraints struct {
Platform []string
Environment []string
Event []string
Branch []string
Status []string
Matrix map[string]string
Platform Constraint
Environment Constraint
Event Constraint
Branch Constraint
Status Constraint
Matrix ConstraintMap
}
//
// // Constraint defines an individual contraint.
// type Constraint struct {
// Include []string
// Exclude []string
// }
//
// // Match returns true if the branch matches the include patterns and does not
// // match any of the exclude patterns.
// func (c *Constraint) Match(v string) bool {
// // when no includes or excludes automatically match
// if len(c.Include) == 0 && len(c.Exclude) == 0 {
// return true
// }
//
// // exclusions are processed first. So we can include everything and then
// // selectively exclude certain sub-patterns.
// for _, pattern := range c.Exclude {
// if pattern == v {
// return false
// }
// if ok, _ := filepath.Match(pattern, v); ok {
// return false
// }
// }
//
// for _, pattern := range c.Include {
// if pattern == v {
// return true
// }
// if ok, _ := filepath.Match(pattern, v); ok {
// return true
// }
// }
//
// return false
// }
// Match returns true if all constraints match the given input. If a single constraint
// fails a false value is returned.
func (c *Constraints) Match(arch, target, event, branch, status string, matrix map[string]string) bool {
return c.Platform.Match(arch) &&
c.Environment.Match(target) &&
c.Event.Match(event) &&
c.Branch.Match(branch) &&
c.Status.Match(status) &&
c.Matrix.Match(matrix)
}
// Constraint defines an individual constraint.
type Constraint struct {
Include []string
Exclude []string
}
// Match returns true if the string matches the include patterns and does not
// match any of the exclude patterns.
func (c *Constraint) Match(v string) bool {
if c.Excludes(v) {
return false
}
if c.Includes(v) {
return true
}
if len(c.Include) == 0 {
return true
}
return false
}
// Includes returns true if the string matches matches the include patterns.
func (c *Constraint) Includes(v string) bool {
for _, pattern := range c.Include {
if ok, _ := filepath.Match(pattern, v); ok {
return true
}
}
return false
}
// Excludes returns true if the string matches matches the exclude patterns.
func (c *Constraint) Excludes(v string) bool {
for _, pattern := range c.Exclude {
if ok, _ := filepath.Match(pattern, v); ok {
return true
}
}
return false
}
// UnmarshalYAML implements custom Yaml unmarshaling.
func (c *Constraint) UnmarshalYAML(unmarshal func(interface{}) error) error {
var out1 = struct {
Include types.StringOrSlice
Exclude types.StringOrSlice
}{}
var out2 types.StringOrSlice
unmarshal(&out1)
unmarshal(&out2)
c.Exclude = out1.Exclude.Slice()
c.Include = append(
out1.Include.Slice(),
out2.Slice()...,
)
return nil
}
// ConstraintMap defines an individual constraint for key value structures.
type ConstraintMap struct {
Include map[string]string
Exclude map[string]string
}
// Match returns true if the params matches the include key values and does not
// match any of the exclude key values.
func (c *ConstraintMap) Match(params map[string]string) bool {
// when no includes or excludes automatically match
if len(c.Include) == 0 && len(c.Exclude) == 0 {
return true
}
// exclusions are processed first. So we can include everything and then
// selectively include others.
if len(c.Exclude) != 0 {
var matches int
for key, val := range c.Exclude {
if params[key] == val {
matches++
}
}
if matches == len(c.Exclude) {
return false
}
}
for key, val := range c.Include {
if params[key] != val {
return false
}
}
return true
}
// UnmarshalYAML implements custom Yaml unmarshaling.
func (c *ConstraintMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
out1 := struct {
Include map[string]string
Exclude map[string]string
}{
Include: map[string]string{},
Exclude: map[string]string{},
}
out2 := map[string]string{}
unmarshal(&out1)
unmarshal(&out2)
c.Include = out1.Include
c.Exclude = out1.Exclude
for k, v := range out2 {
c.Include[k] = v
}
return nil
}

142
yaml/constraint_test.go Normal file
View file

@ -0,0 +1,142 @@
package yaml
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestConstraint(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Constraint", func() {
g.It("Should parse and match emtpy", func() {
c := parseConstraint("")
g.Assert(c.Match("master")).IsTrue()
})
g.It("Should parse and match", func() {
c := parseConstraint("{ include: [ master, develop ] }")
g.Assert(c.Include[0]).Equal("master")
g.Assert(c.Include[1]).Equal("develop")
g.Assert(c.Match("master")).IsTrue()
})
g.It("Should parse and match shortand", func() {
c := parseConstraint("[ master, develop ]")
g.Assert(c.Include[0]).Equal("master")
g.Assert(c.Include[1]).Equal("develop")
g.Assert(c.Match("master")).IsTrue()
})
g.It("Should parse and match shortand string", func() {
c := parseConstraint("master")
g.Assert(c.Include[0]).Equal("master")
g.Assert(c.Match("master")).IsTrue()
})
g.It("Should parse and match exclude", func() {
c := parseConstraint("{ exclude: [ master, develop ] }")
g.Assert(c.Exclude[0]).Equal("master")
g.Assert(c.Exclude[1]).Equal("develop")
g.Assert(c.Match("master")).IsFalse()
})
g.It("Should parse and match exclude shorthand", func() {
c := parseConstraint("{ exclude: master }")
g.Assert(c.Exclude[0]).Equal("master")
g.Assert(c.Match("master")).IsFalse()
})
g.It("Should match include", func() {
b := Constraint{}
b.Include = []string{"master"}
g.Assert(b.Match("master")).IsTrue()
})
g.It("Should match include pattern", func() {
b := Constraint{}
b.Include = []string{"feature/*"}
g.Assert(b.Match("feature/foo")).IsTrue()
})
g.It("Should fail to match include pattern", func() {
b := Constraint{}
b.Include = []string{"feature/*"}
g.Assert(b.Match("master")).IsFalse()
})
g.It("Should match exclude", func() {
b := Constraint{}
b.Exclude = []string{"master"}
g.Assert(b.Match("master")).IsFalse()
})
g.It("Should match exclude pattern", func() {
b := Constraint{}
b.Exclude = []string{"feature/*"}
g.Assert(b.Match("feature/foo")).IsFalse()
})
g.It("Should match when eclude patterns mismatch", func() {
b := Constraint{}
b.Exclude = []string{"foo"}
g.Assert(b.Match("bar")).IsTrue()
})
})
}
func TestConstraintMap(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Constraint Map", func() {
g.It("Should parse and match emtpy", func() {
p := map[string]string{"golang": "1.5", "redis": "3.2"}
c := parseConstraintMap("")
g.Assert(c.Match(p)).IsTrue()
})
g.It("Should parse and match", func() {
p := map[string]string{"golang": "1.5", "redis": "3.2"}
c := parseConstraintMap("{ include: { golang: 1.5 } }")
g.Assert(c.Include["golang"]).Equal("1.5")
g.Assert(c.Match(p)).IsTrue()
})
g.It("Should parse and match shortand", func() {
p := map[string]string{"golang": "1.5", "redis": "3.2"}
c := parseConstraintMap("{ golang: 1.5 }")
g.Assert(c.Include["golang"]).Equal("1.5")
g.Assert(c.Match(p)).IsTrue()
})
g.It("Should parse and match exclude", func() {
p := map[string]string{"golang": "1.5", "redis": "3.2"}
c := parseConstraintMap("{ exclude: { golang: 1.5 } }")
g.Assert(c.Exclude["golang"]).Equal("1.5")
g.Assert(c.Match(p)).IsFalse()
})
g.It("Should parse and mismatch exclude", func() {
p := map[string]string{"golang": "1.5", "redis": "3.2"}
c := parseConstraintMap("{ exclude: { golang: 1.5, redis: 2.8 } }")
g.Assert(c.Exclude["golang"]).Equal("1.5")
g.Assert(c.Exclude["redis"]).Equal("2.8")
g.Assert(c.Match(p)).IsTrue()
})
})
}
func parseConstraint(s string) *Constraint {
c := &Constraint{}
yaml.Unmarshal([]byte(s), c)
return c
}
func parseConstraintMap(s string) *ConstraintMap {
c := &ConstraintMap{}
yaml.Unmarshal([]byte(s), c)
return c
}

View file

@ -85,14 +85,7 @@ type container struct {
Token string `yaml:"registry_token"`
} `yaml:"auth_config"`
Constraints struct {
Platform types.StringOrSlice `yaml:"platform"`
Environment types.StringOrSlice `yaml:"environment"`
Event types.StringOrSlice `yaml:"event"`
Branch types.StringOrSlice `yaml:"branch"`
Status types.StringOrSlice `yaml:"status"`
Matrix map[string]string `yaml:"matrix"`
} `yaml:"when"`
Constraints Constraints `yaml:"when"`
Vargs map[string]interface{} `yaml:",inline"`
}
@ -158,14 +151,7 @@ func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error {
Password: cc.AuthConfig.Password,
Email: cc.AuthConfig.Email,
},
Constraints: Constraints{
Platform: cc.Constraints.Platform.Slice(),
Environment: cc.Constraints.Environment.Slice(),
Event: cc.Constraints.Event.Slice(),
Branch: cc.Constraints.Branch.Slice(),
Status: cc.Constraints.Status.Slice(),
Matrix: cc.Constraints.Matrix,
},
Constraints: cc.Constraints,
})
}
return err

View file

@ -1 +0,0 @@
package interpreter

View file

@ -16,6 +16,7 @@ func Clone(c *yaml.Config, plugin string) error {
Image: plugin,
Name: clone,
}
c.Pipeline = append([]*yaml.Container{s}, c.Pipeline...)
return nil
}

View file

@ -0,0 +1 @@
package transform

View file

@ -14,7 +14,7 @@ import (
func CommandTransform(c *yaml.Config) error {
for _, p := range c.Pipeline {
if len(p.Commands) == 0 {
if isPlugin(p) {
continue
}

View file

@ -0,0 +1,47 @@
package transform
import (
"testing"
"github.com/drone/drone/yaml"
"github.com/franela/goblin"
)
func Test_command(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Command genration", func() {
g.It("should ignore plugin steps", func() {
c := newConfig(&yaml.Container{
Commands: []string{
"go build",
"go test",
},
Vargs: map[string]interface{}{
"depth": 50,
},
})
CommandTransform(c)
g.Assert(len(c.Pipeline[0].Entrypoint)).Equal(0)
g.Assert(len(c.Pipeline[0].Command)).Equal(0)
g.Assert(c.Pipeline[0].Environment["DRONE_SCRIPT"]).Equal("")
})
g.It("should set entrypoint, command and environment variables", func() {
c := newConfig(&yaml.Container{
Commands: []string{
"go build",
"go test",
},
})
CommandTransform(c)
g.Assert(c.Pipeline[0].Entrypoint).Equal([]string{"/bin/sh", "-c"})
g.Assert(c.Pipeline[0].Command).Equal([]string{"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e"})
g.Assert(c.Pipeline[0].Environment["DRONE_SCRIPT"] != "").IsTrue()
})
})
}

View file

@ -0,0 +1,27 @@
package transform
import (
"testing"
"github.com/drone/drone/yaml"
"github.com/franela/goblin"
)
func Test_env(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("environment variables", func() {
g.It("should be copied", func() {
envs := map[string]string{"CI": "drone"}
c := newConfig(&yaml.Container{
Environment: map[string]string{},
})
Environ(c, envs)
g.Assert(c.Pipeline[0].Environment["CI"]).Equal("drone")
})
})
}

58
yaml/transform/filter.go Normal file
View file

@ -0,0 +1,58 @@
package transform
import (
"github.com/drone/drone/model"
"github.com/drone/drone/yaml"
)
// DefaultFilter is a transform function that applies default Filters to each
// step in the Yaml specification file.
func DefaultFilter(conf *yaml.Config) {
for _, step := range conf.Pipeline {
defaultStatus(step)
defaultEvent(step)
}
}
// defaultStatus sets default status conditions.
func defaultStatus(c *yaml.Container) {
if !isEmpty(c.Constraints.Status) {
return
}
c.Constraints.Status.Include = []string{
model.StatusSuccess,
}
}
// defaultEvent sets default event conditions.
func defaultEvent(c *yaml.Container) {
if !isEmpty(c.Constraints.Event) {
return
}
if isPlugin(c) && !isClone(c) {
c.Constraints.Event.Exclude = []string{
model.EventPull,
}
}
}
// helper function returns true if the step is a clone step.
func isEmpty(c yaml.Constraint) bool {
return len(c.Include) == 0 && len(c.Exclude) == 0
}
// helper function returns true if the step is a plugin step.
func isPlugin(c *yaml.Container) bool {
return len(c.Commands) == 0 || len(c.Vargs) != 0
}
// helper function returns true if the step is a command step.
func isCommand(c *yaml.Container) bool {
return len(c.Commands) != 0
}
// helper function returns true if the step is a clone step.
func isClone(c *yaml.Container) bool {
return c.Name == "clone"
}

View file

@ -7,9 +7,10 @@ import (
"github.com/drone/drone/yaml"
)
// ImagePull transforms the Yaml to automatically pull the latest image.
func ImagePull(conf *yaml.Config, pull bool) error {
for _, plugin := range conf.Pipeline {
if len(plugin.Commands) == 0 || len(plugin.Vargs) == 0 {
if !isPlugin(plugin) {
continue
}
plugin.Pull = pull
@ -17,6 +18,7 @@ func ImagePull(conf *yaml.Config, pull bool) error {
return nil
}
// ImageTag transforms the Yaml to use the :latest image tag when empty.
func ImageTag(conf *yaml.Config) error {
for _, image := range conf.Pipeline {
if !strings.Contains(image.Image, ":") {
@ -31,6 +33,7 @@ func ImageTag(conf *yaml.Config) error {
return nil
}
// ImageName transforms the Yaml to replace underscores with dashes.
func ImageName(conf *yaml.Config) error {
for _, image := range conf.Pipeline {
image.Image = strings.Replace(image.Image, "_", "-", -1)
@ -38,12 +41,13 @@ func ImageName(conf *yaml.Config) error {
return nil
}
// ImageNamespace transforms the Yaml to use a default namepsace for plugins.
func ImageNamespace(conf *yaml.Config, namespace string) error {
for _, image := range conf.Pipeline {
if strings.Contains(image.Image, "/") {
continue
}
if len(image.Vargs) == 0 {
if !isPlugin(image) {
continue
}
image.Image = filepath.Join(namespace, image.Image)
@ -51,6 +55,8 @@ func ImageNamespace(conf *yaml.Config, namespace string) error {
return nil
}
// ImageEscalate transforms the Yaml to automatically enable privileged mode
// for a subset of white-listed plugins matching the given patterns.
func ImageEscalate(conf *yaml.Config, patterns []string) error {
for _, c := range conf.Pipeline {
for _, pattern := range patterns {

View file

@ -0,0 +1,50 @@
package transform
import (
"testing"
"github.com/drone/drone/yaml"
"github.com/franela/goblin"
)
func Test_pull(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("pull image", func() {
g.It("should be enabled for plugins", func() {
c := newConfig(&yaml.Container{})
ImagePull(c, true)
g.Assert(c.Pipeline[0].Pull).IsTrue()
})
g.It("should be disabled for plugins", func() {
c := newConfig(&yaml.Container{})
ImagePull(c, false)
g.Assert(c.Pipeline[0].Pull).IsFalse()
})
g.It("should not apply to commands", func() {
c := newConfig(&yaml.Container{
Commands: []string{
"go build",
"go test",
},
})
ImagePull(c, true)
g.Assert(c.Pipeline[0].Pull).IsFalse()
})
g.It("should not apply to services", func() {
c := newConfigService(&yaml.Container{
Image: "mysql",
})
ImagePull(c, true)
g.Assert(c.Services[0].Pull).IsFalse()
})
})
}

View file

@ -1,80 +1,47 @@
package transform
import "github.com/drone/drone/yaml"
import (
"path/filepath"
// PluginDisable disables plugins. This is intended for use when executing the
// pipeline locally on your own computer.
func PluginDisable(conf *yaml.Config, disabled bool) {
"github.com/drone/drone/yaml"
)
// PluginDisable is a transform function that alters the Yaml configuration to
// disables plugins. This is intended for use when executing the pipeline
// locally on your own computer.
func PluginDisable(conf *yaml.Config, patterns []string) error {
for _, container := range conf.Pipeline {
if len(container.Vargs) != 0 || container.Name == "clone" {
container.Disabled = disabled
if len(container.Commands) != 0 { // skip build steps
continue
}
var match bool
for _, pattern := range patterns {
if ok, _ := filepath.Match(pattern, container.Name); ok {
match = true
break
}
}
if !match {
container.Disabled = true
}
}
return nil
}
//
// import (
// "fmt"
// "reflect"
// "strconv"
// "strings"
//
// "github.com/drone/drone/yaml"
// "github.com/libcd/libyaml/parse"
//
// json "github.com/ghodss/yaml"
// "gopkg.in/yaml.v2"
// )
//
// func
//
// // argsToEnv uses reflection to convert a map[string]interface to a list
// // of environment variables.
// func argsToEnv(from map[string]interface{}, to map[string]string) error {
//
// for k, v := range from {
// t := reflect.TypeOf(v)
// vv := reflect.ValueOf(v)
//
// k = "PLUGIN_" + strings.ToUpper(k)
//
// switch t.Kind() {
// case reflect.Bool:
// to[k] = strconv.FormatBool(vv.Bool())
//
// case reflect.String:
// to[k] = vv.String()
//
// case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
// to[k] = fmt.Sprintf("%v", vv.Int())
//
// case reflect.Float32, reflect.Float64:
// to[k] = fmt.Sprintf("%v", vv.Float())
//
// case reflect.Map:
// yml, _ := yaml.Marshal(vv.Interface())
// out, _ := json.YAMLToJSON(yml)
// to[k] = string(out)
//
// case reflect.Slice:
// out, err := yaml.Marshal(vv.Interface())
// if err != nil {
// return err
// }
//
// in := []string{}
// err := yaml.Unmarshal(out, &in)
// if err == nil {
// to[k] = strings.Join(in, ",")
// } else {
// out, err = json.YAMLToJSON(out)
// if err != nil {
// return err
// }
// to[k] = string(out)
// }
// }
// }
//
// return nil
// }
// PluginParams is a transform function that alters the Yaml configuration to
// include plugin parameters as environment variables.
func PluginParams(conf *yaml.Config) error {
for _, container := range conf.Pipeline {
if len(container.Vargs) == 0 {
continue
}
if container.Environment == nil {
container.Environment = map[string]string{}
}
err := argsToEnv(container.Vargs, container.Environment)
if err != nil {
return err
}
}
return nil
}

View file

@ -0,0 +1 @@
package transform

View file

@ -18,13 +18,14 @@ func Pod(c *yaml.Config) error {
)
ambassador := &yaml.Container{
ID: fmt.Sprintf("drone_ambassador_%s", rand),
Name: "ambassador",
Image: "busybox:latest",
Detached: true,
Entrypoint: []string{"/bin/sleep"},
Command: []string{"86400"},
Volumes: []string{c.Workspace.Path, c.Workspace.Base},
ID: fmt.Sprintf("drone_ambassador_%s", rand),
Name: "ambassador",
Image: "busybox:latest",
Detached: true,
Entrypoint: []string{"/bin/sleep"},
Command: []string{"86400"},
Volumes: []string{c.Workspace.Path, c.Workspace.Base},
Environment: map[string]string{},
}
network := fmt.Sprintf("container:%s", ambassador.ID)

View file

@ -5,27 +5,35 @@ import (
"github.com/drone/drone/yaml"
)
func Secret(c *yaml.Config, event string, secrets []*model.Secret) error {
func ImageSecrets(c *yaml.Config, secrets []*model.Secret, event string) error {
var images []*yaml.Container
images = append(images, c.Pipeline...)
images = append(images, c.Services...)
for _, p := range c.Pipeline {
for _, secret := range secrets {
switch secret.Name {
case "REGISTRY_USERNAME":
p.AuthConfig.Username = secret.Value
case "REGISTRY_PASSWORD":
p.AuthConfig.Password = secret.Value
case "REGISTRY_EMAIL":
p.AuthConfig.Email = secret.Value
default:
if p.Environment == nil {
p.Environment = map[string]string{}
}
p.Environment[secret.Name] = secret.Value
}
}
for _, image := range images {
imageSecrets(image, secrets, event)
}
return nil
}
func imageSecrets(c *yaml.Container, secrets []*model.Secret, event string) {
for _, secret := range secrets {
if !secret.Match(c.Image, event) {
continue
}
switch secret.Name {
case "REGISTRY_USERNAME":
c.AuthConfig.Username = secret.Value
case "REGISTRY_PASSWORD":
c.AuthConfig.Password = secret.Value
case "REGISTRY_EMAIL":
c.AuthConfig.Email = secret.Value
default:
if c.Environment == nil {
c.Environment = map[string]string{}
}
c.Environment[secret.Name] = secret.Value
}
}
}

View file

@ -0,0 +1 @@
package transform

62
yaml/transform/util.go Normal file
View file

@ -0,0 +1,62 @@
package transform
import (
"fmt"
"reflect"
"strconv"
"strings"
json "github.com/ghodss/yaml"
"gopkg.in/yaml.v2"
)
// argsToEnv uses reflection to convert a map[string]interface to a list
// of environment variables.
func argsToEnv(from map[string]interface{}, to map[string]string) error {
for k, v := range from {
t := reflect.TypeOf(v)
vv := reflect.ValueOf(v)
k = "PLUGIN_" + strings.ToUpper(k)
switch t.Kind() {
case reflect.Bool:
to[k] = strconv.FormatBool(vv.Bool())
case reflect.String:
to[k] = vv.String()
case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
to[k] = fmt.Sprintf("%v", vv.Int())
case reflect.Float32, reflect.Float64:
to[k] = fmt.Sprintf("%v", vv.Float())
case reflect.Map:
yml, _ := yaml.Marshal(vv.Interface())
out, _ := json.YAMLToJSON(yml)
to[k] = string(out)
case reflect.Slice:
out, err := yaml.Marshal(vv.Interface())
if err != nil {
return err
}
in := []string{}
err = yaml.Unmarshal(out, &in)
if err == nil {
to[k] = strings.Join(in, ",")
} else {
out, err = json.YAMLToJSON(out)
if err != nil {
return err
}
to[k] = string(out)
}
}
}
return nil
}

View file

@ -11,7 +11,7 @@ func Check(c *yaml.Config, trusted bool) error {
images = append(images, c.Pipeline...)
images = append(images, c.Services...)
for _, image := range images {
for _, image := range c.Pipeline {
if err := CheckEntrypoint(image); err != nil {
return err
}
@ -22,15 +22,20 @@ func Check(c *yaml.Config, trusted bool) error {
return err
}
}
for _, image := range c.Services {
if trusted {
continue
}
if err := CheckTrusted(image); err != nil {
return err
}
}
return nil
}
// validate the plugin command and entrypoint and return an error
// the user attempts to set or override these values.
func CheckEntrypoint(c *yaml.Container) error {
if len(c.Vargs) == 0 {
return nil
}
if len(c.Entrypoint) != 0 {
return fmt.Errorf("Cannot set plugin Entrypoint")
}

View file

@ -0,0 +1,154 @@
package transform
import (
"testing"
"github.com/drone/drone/yaml"
"github.com/franela/goblin"
)
func Test_validate(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("validating", func() {
g.Describe("privileged attributes", func() {
g.It("should not error when trusted build", func() {
c := newConfig(&yaml.Container{Privileged: true})
err := Check(c, true)
g.Assert(err == nil).IsTrue("error should be nil")
})
g.It("should error when privleged mode", func() {
c := newConfig(&yaml.Container{
Privileged: true,
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode")
})
g.It("should error when privleged service container", func() {
c := newConfigService(&yaml.Container{
Privileged: true,
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode")
})
g.It("should error when dns configured", func() {
c := newConfig(&yaml.Container{
DNS: []string{"8.8.8.8"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use custom dns")
})
g.It("should error when dns_search configured", func() {
c := newConfig(&yaml.Container{
DNSSearch: []string{"8.8.8.8"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use dns_search")
})
g.It("should error when devices configured", func() {
c := newConfig(&yaml.Container{
Devices: []string{"/dev/foo"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use devices")
})
g.It("should error when extra_hosts configured", func() {
c := newConfig(&yaml.Container{
ExtraHosts: []string{"1.2.3.4 foo.com"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use extra_hosts")
})
g.It("should error when network configured", func() {
c := newConfig(&yaml.Container{
Network: "host",
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to override the network")
})
g.It("should error when oom_kill_disabled configured", func() {
c := newConfig(&yaml.Container{
OomKillDisable: true,
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to disable oom_kill")
})
g.It("should error when volumes configured", func() {
c := newConfig(&yaml.Container{
Volumes: []string{"/:/tmp"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use volumes")
})
g.It("should error when volumes_from configured", func() {
c := newConfig(&yaml.Container{
VolumesFrom: []string{"drone"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use volumes_from")
})
})
g.Describe("plugin configuration", func() {
g.It("should error when entrypoint is configured", func() {
c := newConfig(&yaml.Container{
Entrypoint: []string{"/bin/sh"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Cannot set plugin Entrypoint")
})
g.It("should error when command is configured", func() {
c := newConfig(&yaml.Container{
Command: []string{"cat", "/proc/1/status"},
})
err := Check(c, false)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Cannot set plugin Command")
})
g.It("should not error when empty entrypoint, command", func() {
c := newConfig(&yaml.Container{})
err := Check(c, false)
g.Assert(err == nil).IsTrue("error should be nil")
})
})
})
}
func newConfig(container *yaml.Container) *yaml.Config {
return &yaml.Config{
Pipeline: []*yaml.Container{container},
}
}
func newConfigService(container *yaml.Container) *yaml.Config {
return &yaml.Config{
Services: []*yaml.Container{container},
}
}

View file

@ -0,0 +1,99 @@
package transform
import (
"testing"
"github.com/franela/goblin"
"github.com/drone/drone/yaml"
)
func TestWorkspace(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("workspace", func() {
defaultBase := "/go"
defaultPath := "src/github.com/octocat/hello-world"
g.It("should not override user paths", func() {
base := "/drone"
path := "/drone/src/github.com/octocat/hello-world"
conf := &yaml.Config{
Workspace: &yaml.Workspace{
Base: base,
Path: path,
},
}
WorkspaceTransform(conf, defaultBase, defaultPath)
g.Assert(conf.Workspace.Base).Equal(base)
g.Assert(conf.Workspace.Path).Equal(path)
})
g.It("should convert user paths to absolute", func() {
base := "/drone"
path := "src/github.com/octocat/hello-world"
abs := "/drone/src/github.com/octocat/hello-world"
conf := &yaml.Config{
Workspace: &yaml.Workspace{
Base: base,
Path: path,
},
}
WorkspaceTransform(conf, defaultBase, defaultPath)
g.Assert(conf.Workspace.Base).Equal(base)
g.Assert(conf.Workspace.Path).Equal(abs)
})
g.It("should set the default path", func() {
var base = "/go"
var path = "/go/src/github.com/octocat/hello-world"
conf := &yaml.Config{}
WorkspaceTransform(conf, defaultBase, defaultPath)
g.Assert(conf.Workspace.Base).Equal(base)
g.Assert(conf.Workspace.Path).Equal(path)
})
g.It("should use workspace as working_dir", func() {
var base = "/drone"
var path = "/drone/src/github.com/octocat/hello-world"
conf := &yaml.Config{
Workspace: &yaml.Workspace{
Base: base,
Path: path,
},
Pipeline: []*yaml.Container{
{},
},
}
WorkspaceTransform(conf, defaultBase, defaultPath)
g.Assert(conf.Pipeline[0].WorkingDir).Equal(path)
})
g.It("should not use workspace as working_dir for services", func() {
var base = "/drone"
var path = "/drone/src/github.com/octocat/hello-world"
conf := &yaml.Config{
Workspace: &yaml.Workspace{
Base: base,
Path: path,
},
Services: []*yaml.Container{
{},
},
}
WorkspaceTransform(conf, defaultBase, defaultPath)
g.Assert(conf.Services[0].WorkingDir).Equal("")
})
})
}

View file

@ -36,3 +36,8 @@ func (s *MapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (s *MapEqualSlice) Map() map[string]string {
return s.parts
}
// NewMapEqualSlice returns a new MapEqualSlice.
func NewMapEqualSlice(from map[string]string) *MapEqualSlice {
return &MapEqualSlice{from}
}

View file

@ -28,3 +28,8 @@ func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
func (s StringOrSlice) Slice() []string {
return s.parts
}
// NewStringOrSlice returns a new StringOrSlice.
func NewStringOrSlice(from []string) *StringOrSlice {
return &StringOrSlice{from}
}