still a wip. container that is launched to run a build

cmd/drone-build/Dockerfile

@@ -0,0 +1,18 @@
+# Docker image for Drone's git-clone plugin
+#
+#     docker build --rm=true -t drone/drone-build .
+
+FROM library/golang:1.4
+
+# copy the local package files to the container's workspace.
+#ADD . /go/src/github.com/drone/drone-build/
+
+# build the program inside the container.
+#RUN go get github.com/drone/drone-build/... && \
+#    go install github.com/drone/drone-build
+
+
+ADD drone-build /go/bin/
+
+# run the git-clone plugin when the container starts
+ENTRYPOINT ["/go/bin/drone-build"]

cmd/drone-build/client.go

@@ -0,0 +1,203 @@
+package main
+
+import (
+	"errors"
+	"os"
+
+	"github.com/samalba/dockerclient"
+)
+
+var (
+	ErrTimeout = errors.New("Timeout")
+	ErrLogging = errors.New("Logs not available")
+)
+
+var (
+	// options to fetch the stdout and stderr logs
+	logOpts = &dockerclient.LogOptions{
+		Stdout: true,
+		Stderr: true,
+	}
+
+	// options to fetch the stdout and stderr logs
+	// by tailing the output.
+	logOptsTail = &dockerclient.LogOptions{
+		Follow: true,
+		Stdout: true,
+		Stderr: true,
+	}
+)
+
+// client is a wrapper around the default Docker client
+// that tracks all created containers ensures some default
+// configurations are in place.
+type client struct {
+	dockerclient.Client
+	info  *dockerclient.ContainerInfo
+	names []string // names of created containers
+}
+
+func newClient(docker dockerclient.Client) (*client, error) {
+
+	// creates an ambassador container
+	conf := &dockerclient.ContainerConfig{}
+	conf.HostConfig = dockerclient.HostConfig{}
+	conf.Entrypoint = []string{"/bin/sleep"}
+	conf.Cmd = []string{"86400"}
+	conf.Image = "busybox"
+	conf.Volumes = map[string]struct{}{}
+	conf.Volumes["/drone"] = struct{}{}
+	info, err := daemon(docker, conf, false)
+	if err != nil {
+		return nil, err
+	}
+
+	return &client{Client: docker, info: info}, nil
+}
+
+// CreateContainer creates a container and internally
+// caches its container id.
+func (c *client) CreateContainer(conf *dockerclient.ContainerConfig, name string) (string, error) {
+	conf.Env = append(conf.Env, "affinity:container=="+c.info.Id)
+	id, err := c.Client.CreateContainer(conf, name)
+	if err == nil {
+		c.names = append(c.names, id)
+	}
+	return id, err
+}
+
+// StartContainer starts a container and links to an
+// ambassador container sharing the build machiens volume.
+func (c *client) StartContainer(id string, conf *dockerclient.HostConfig) error {
+	conf.VolumesFrom = append(conf.VolumesFrom, c.info.Id)
+	if len(conf.NetworkMode) == 0 {
+		conf.NetworkMode = "container:" + c.info.Id
+	}
+	return c.Client.StartContainer(id, conf)
+}
+
+// Destroy will terminate and destroy all containers that
+// were created by this client.
+func (c *client) Destroy() error {
+	for _, id := range c.names {
+		c.Client.KillContainer(id, "9")
+		c.Client.RemoveContainer(id, true, true)
+	}
+	c.Client.KillContainer(c.info.Id, "9")
+	return c.Client.RemoveContainer(c.info.Id, true, true)
+}
+
+func run(client dockerclient.Client, conf *dockerclient.ContainerConfig, pull bool) (*dockerclient.ContainerInfo, error) {
+	// force-pull the image if specified.
+	// TEMPORARY while we are in beta mode we should always re-pull drone plugins
+	if pull { //|| strings.HasPrefix(conf.Image, "plugins/") {
+		client.PullImage(conf.Image, nil)
+	}
+
+	// attempts to create the contianer
+	id, err := client.CreateContainer(conf, "")
+	if err != nil {
+		// and pull the image and re-create if that fails
+		client.PullImage(conf.Image, nil)
+		id, err = client.CreateContainer(conf, "")
+		// make sure the container is removed in
+		// the event of a creation error.
+		if len(id) != 0 {
+			client.RemoveContainer(id, true, true)
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// ensures the container is always stopped
+	// and ready to be removed.
+	defer func() {
+		client.StopContainer(id, 5)
+		client.KillContainer(id, "9")
+	}()
+
+	// fetches the container information.
+	info, err := client.InspectContainer(id)
+	if err != nil {
+		client.RemoveContainer(id, true, true)
+		return nil, err
+	}
+
+	// channel listening for errors while the
+	// container is running async.
+	errc := make(chan error, 1)
+	infoc := make(chan *dockerclient.ContainerInfo, 1)
+	go func() {
+
+		// starts the container
+		err := client.StartContainer(id, &conf.HostConfig)
+		if err != nil {
+			errc <- err
+			return
+		}
+
+		// blocks and waits for the container to finish
+		// by streaming the logs (to /dev/null). Ideally
+		// we could use the `wait` function instead
+		rc, err := client.ContainerLogs(id, logOptsTail)
+		if err != nil {
+			errc <- err
+			return
+		}
+		defer rc.Close()
+		StdCopy(os.Stdout, os.Stdout, rc)
+
+		// fetches the container information
+		info, err := client.InspectContainer(id)
+		if err != nil {
+			errc <- err
+			return
+		}
+		infoc <- info
+	}()
+
+	select {
+	case info := <-infoc:
+		return info, nil
+	case err := <-errc:
+		return info, err
+		// TODO checkout net.Context and cancel
+		// case <-time.After(timeout):
+		// 	return info, ErrTimeout
+	}
+}
+
+func daemon(client dockerclient.Client, conf *dockerclient.ContainerConfig, pull bool) (*dockerclient.ContainerInfo, error) {
+	// force-pull the image
+	if pull {
+		client.PullImage(conf.Image, nil)
+	}
+
+	// attempts to create the contianer
+	id, err := client.CreateContainer(conf, "")
+	if err != nil {
+		// and pull the image and re-create if that fails
+		client.PullImage(conf.Image, nil)
+		id, err = client.CreateContainer(conf, "")
+		// make sure the container is removed in
+		// the event of a creation error.
+		if len(id) != 0 {
+			client.RemoveContainer(id, true, true)
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// fetches the container information
+	info, err := client.InspectContainer(id)
+	if err != nil {
+		client.RemoveContainer(id, true, true)
+		return nil, err
+	}
+
+	// starts the container
+	err = client.StartContainer(id, &conf.HostConfig)
+	return info, err
+}

cmd/drone-build/copy.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+const (
+	StdWriterPrefixLen = 8
+	StdWriterFdIndex   = 0
+	StdWriterSizeIndex = 4
+)
+
+type StdType [StdWriterPrefixLen]byte
+
+var (
+	Stdin  StdType = StdType{0: 0}
+	Stdout StdType = StdType{0: 1}
+	Stderr StdType = StdType{0: 2}
+)
+
+type StdWriter struct {
+	io.Writer
+	prefix  StdType
+	sizeBuf []byte
+}
+
+var ErrInvalidStdHeader = errors.New("Unrecognized input header")
+
+// StdCopy is a modified version of io.Copy.
+//
+// StdCopy will demultiplex `src`, assuming that it contains two streams,
+// previously multiplexed together using a StdWriter instance.
+// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
+//
+// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
+// In other words: if `err` is non nil, it indicates a real underlying error.
+//
+// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
+func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
+	var (
+		buf       = make([]byte, 32*1024+StdWriterPrefixLen+1)
+		bufLen    = len(buf)
+		nr, nw    int
+		er, ew    error
+		out       io.Writer
+		frameSize int
+	)
+
+	for {
+		// Make sure we have at least a full header
+		for nr < StdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < StdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Check the first byte to know where to write
+		switch buf[StdWriterFdIndex] {
+		case 0:
+			fallthrough
+		case 1:
+			// Write on stdout
+			out = dstout
+		case 2:
+			// Write on stderr
+			out = dsterr
+		default:
+			return 0, ErrInvalidStdHeader
+		}
+
+		// Retrieve the size of the frame
+		frameSize = int(binary.BigEndian.Uint32(buf[StdWriterSizeIndex : StdWriterSizeIndex+4]))
+
+		// Check if the buffer is big enough to read the frame.
+		// Extend it if necessary.
+		if frameSize+StdWriterPrefixLen > bufLen {
+			buf = append(buf, make([]byte, frameSize+StdWriterPrefixLen-bufLen+1)...)
+			bufLen = len(buf)
+		}
+
+		// While the amount of bytes read is less than the size of the frame + header, we keep reading
+		for nr < frameSize+StdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < frameSize+StdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Write the retrieved frame (without header)
+		nw, ew = out.Write(buf[StdWriterPrefixLen : frameSize+StdWriterPrefixLen])
+		if ew != nil {
+			return 0, ew
+		}
+		// If the frame has not been fully written: error
+		if nw != frameSize {
+			return 0, io.ErrShortWrite
+		}
+		written += int64(nw)
+
+		// Move the rest of the buffer to the beginning
+		copy(buf, buf[frameSize+StdWriterPrefixLen:])
+		// Move the index
+		nr -= frameSize + StdWriterPrefixLen
+	}
+}

cmd/drone-build/main.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/drone/drone/common"
+	"github.com/samalba/dockerclient"
+)
+
+var (
+	clone   = flag.Bool("clone", false, "")
+	build   = flag.Bool("build", false, "")
+	publish = flag.Bool("publish", false, "")
+	deploy  = flag.Bool("deploy", false, "")
+	notify  = flag.Bool("notify", false, "")
+)
+
+func main() {
+	flag.Parse()
+
+	ctx, err := parseContext()
+	if err != nil {
+		fmt.Println("Error launching build container.", err)
+		os.Exit(1)
+		return
+	}
+	createClone(ctx)
+
+	// creates the Docker client, connecting to the
+	// linked Docker daemon
+	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
+	if err != nil {
+		fmt.Println("Error connecting to build server.", err)
+		os.Exit(1)
+		return
+	}
+
+	// creates a wrapper Docker client that uses an ambassador
+	// container to create a pod-like environment.
+	client, err := newClient(docker)
+	if err != nil {
+		fmt.Println("Error starting build server pod", err)
+		os.Exit(1)
+	}
+	ctx.client = client
+	defer client.Destroy()
+
+	// performs some initial parsing and pre-processing steps
+	// prior to executing our build tasks.
+	err = setup(ctx)
+	if err != nil {
+		fmt.Println("Error processing .drone.yml file.", err)
+		client.Destroy()
+		os.Exit(1)
+	}
+
+	var execs []execFunc
+	if *clone {
+		execs = append(execs, execClone)
+	}
+	if *build {
+		execs = append(execs, execSetup)
+		execs = append(execs, execCompose)
+		execs = append(execs, execBuild)
+	}
+	if *publish {
+		execs = append(execs, execPublish)
+	}
+	if *deploy {
+		execs = append(execs, execDeploy)
+	}
+
+	// Loop through and execute each step.
+	for i, exec_ := range execs {
+		code, err := exec_(ctx)
+		if err != nil {
+			fmt.Printf("00%d Error executing build\n", i+1)
+			fmt.Println(err)
+			code = 255
+		}
+		if code != 0 {
+			ctx.Build.ExitCode = code
+			break
+		}
+	}
+
+	// Optionally execute notification steps.
+	if *notify {
+		execNotify(ctx)
+	}
+
+	client.Destroy()
+	os.Exit(ctx.Build.ExitCode)
+}
+
+func createClone(c *Context) error {
+	c.Clone = &common.Clone{
+		Netrc:   c.Netrc,
+		Keypair: c.Keys,
+		Remote:  c.Repo.Clone,
+		Origin:  c.Repo.Clone,
+	}
+
+	c.Clone.Origin = c.Repo.Clone
+	c.Clone.Remote = c.Repo.Clone
+	c.Clone.Sha = c.Commit.Sha
+	c.Clone.Ref = c.Commit.Ref
+	c.Clone.Branch = c.Commit.Branch
+	// TODO move this to the main app (github package)
+	if strings.HasPrefix(c.Clone.Branch, "refs/heads/") {
+		c.Clone.Branch = c.Clone.Branch[11:]
+	}
+
+	// TODO we should also pass the SourceSha, SourceBranch, etc
+	// to the clone object for merge requests from bitbucket, gitlab, et al
+	// if len(c.Commit.PullRequest) != 0 {
+	// }
+
+	url_, err := url.Parse(c.Repo.Link)
+	if err != nil {
+		return err
+	}
+	c.Clone.Dir = filepath.Join("/drone/src", url_.Host, c.Repo.FullName)
+
+	// attempt to extract the clone path. i'm not a huge fan of
+	// this, by the way, but for now we'll keep it.
+	// TODO consider moving this to a transform?
+	pathv, ok := c.Conf.Clone.Config["path"]
+	if ok {
+		path, ok := pathv.(string)
+		if ok {
+			c.Clone.Dir = filepath.Join("/drone/src", path)
+		}
+	}
+	return nil
+}
+
+func parseContext() (*Context, error) {
+	c := &Context{}
+	for i, arg := range os.Args {
+		if arg == "--" && len(os.Args) != i+1 {
+			buf := bytes.NewBufferString(os.Args[i+1])
+			err := json.NewDecoder(buf).Decode(c)
+			return c, err
+		}
+	}
+	err := json.NewDecoder(os.Stdin).Decode(c)
+	return c, err
+}

cmd/drone-build/run.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"encoding/base64"
+	"fmt"
+
+	"github.com/drone/drone/common"
+	"github.com/drone/drone/parser"
+	"github.com/drone/drone/parser/inject"
+	"github.com/samalba/dockerclient"
+)
+
+type Context struct {
+	// Links  *common.Link
+	Clone  *common.Clone   `json:"clone"`
+	Repo   *common.Repo    `json:"repo"`
+	Commit *common.Commit  `json:"commit"`
+	Build  *common.Build   `json:"build"`
+	Keys   *common.Keypair `json:"keys"`
+	Netrc  *common.Netrc   `json:"netrc"`
+	Yaml   []byte          `json:"yaml"`
+	Conf   *common.Config  `json:"-"`
+	infos  []*dockerclient.ContainerInfo
+	client dockerclient.Client
+}
+
+func setup(c *Context) error {
+	var err error
+
+	// inject the matrix parameters into the yaml
+	injected := inject.Inject(string(c.Yaml), c.Build.Environment)
+	c.Conf, err = parser.ParseSingle(injected, parser.DefaultOpts)
+	if err != nil {
+		return err
+	}
+
+	// and append the matrix parameters as environment
+	// variables for the build
+	for k, v := range c.Build.Environment {
+		env := k + "=" + v
+		c.Conf.Build.Environment = append(c.Conf.Build.Environment, env)
+	}
+
+	// and append drone, jenkins, travis and other
+	// environment variables that may be of use.
+	for k, v := range toEnv(c) {
+		env := k + "=" + v
+		c.Conf.Build.Environment = append(c.Conf.Build.Environment, env)
+	}
+
+	return nil
+}
+
+type execFunc func(c *Context) (int, error)
+
+func execClone(c *Context) (int, error) {
+	conf := toContainerConfig(c.Conf.Clone)
+	conf.Cmd = toCommand(c, c.Conf.Clone)
+	info, err := run(c.client, conf, c.Conf.Clone.Pull)
+	if err != nil {
+		return 255, err
+	}
+	return info.State.ExitCode, nil
+}
+
+func execBuild(c *Context) (int, error) {
+	conf := toContainerConfig(c.Conf.Build)
+	conf.Entrypoint = []string{"/bin/bash", "-e"}
+	conf.Cmd = []string{"/drone/bin/build.sh"}
+	info, err := run(c.client, conf, c.Conf.Build.Pull)
+	if err != nil {
+		return 255, err
+	}
+	return info.State.ExitCode, nil
+}
+
+func execSetup(c *Context) (int, error) {
+	conf := toContainerConfig(c.Conf.Setup)
+	conf.Cmd = toCommand(c, c.Conf.Setup)
+	info, err := run(c.client, conf, c.Conf.Setup.Pull)
+	if err != nil {
+		return 255, err
+	}
+	return info.State.ExitCode, nil
+}
+
+func execDeploy(c *Context) (int, error) {
+	return runSteps(c, c.Conf.Deploy)
+}
+
+func execPublish(c *Context) (int, error) {
+	return runSteps(c, c.Conf.Publish)
+}
+
+func execNotify(c *Context) (int, error) {
+	return runSteps(c, c.Conf.Notify)
+}
+
+func execCompose(c *Context) (int, error) {
+	for _, step := range c.Conf.Compose {
+		conf := toContainerConfig(step)
+		_, err := daemon(c.client, conf, step.Pull)
+		if err != nil {
+			return 0, err
+		}
+	}
+	return 0, nil
+}
+
+func trace(s string) string {
+	cmd := fmt.Sprintf("$ %s\n", s)
+	encoded := base64.StdEncoding.EncodeToString([]byte(cmd))
+	return fmt.Sprintf("echo %s | base64 --decode\n", encoded)
+}
+
+func newline(s string) string {
+	return fmt.Sprintf("%s\n", s)
+}
+
+func runSteps(c *Context, steps map[string]*common.Step) (int, error) {
+	for _, step := range steps {
+
+		// verify the step matches the branch
+		// and other specifications
+		if step.Condition == nil ||
+			!step.Condition.MatchOwner(c.Repo.Owner) ||
+			!step.Condition.MatchBranch(c.Clone.Branch) ||
+			!step.Condition.MatchMatrix(c.Build.Environment) {
+			continue
+		}
+
+		conf := toContainerConfig(step)
+		conf.Cmd = toCommand(c, step)
+		info, err := run(c.client, conf, step.Pull)
+		if err != nil {
+			return 255, err
+		} else if info.State.ExitCode != 0 {
+			return info.State.ExitCode, nil
+		}
+	}
+	return 0, nil
+}

cmd/drone-build/util.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"encoding/json"
+	"strconv"
+	"strings"
+
+	"github.com/drone/drone/common"
+	"github.com/samalba/dockerclient"
+)
+
+// helper function that converts the build step to
+// a containerConfig for use with the dockerclient
+func toContainerConfig(step *common.Step) *dockerclient.ContainerConfig {
+	config := &dockerclient.ContainerConfig{
+		Image:      step.Image,
+		Env:        step.Environment,
+		Cmd:        step.Command,
+		Entrypoint: step.Entrypoint,
+		WorkingDir: step.WorkingDir,
+		HostConfig: dockerclient.HostConfig{
+			Privileged:  step.Privileged,
+			NetworkMode: step.NetworkMode,
+		},
+	}
+
+	config.Volumes = map[string]struct{}{}
+	for _, path := range step.Volumes {
+		if strings.Index(path, ":") == -1 {
+			continue
+		}
+		parts := strings.Split(path, ":")
+		config.Volumes[parts[1]] = struct{}{}
+		config.HostConfig.Binds = append(config.HostConfig.Binds, path)
+	}
+
+	return config
+}
+
+// helper function to inject drone-specific environment
+// variables into the container.
+func toEnv(c *Context) map[string]string {
+	return map[string]string{
+		"CI":           "true",
+		"BUILD_DIR":    c.Clone.Dir,
+		"BUILD_ID":     strconv.Itoa(c.Commit.Sequence),
+		"BUILD_NUMBER": strconv.Itoa(c.Commit.Sequence),
+		"JOB_NAME":     c.Repo.FullName,
+		"WORKSPACE":    c.Clone.Dir,
+		"GIT_BRANCH":   c.Clone.Branch,
+		"GIT_COMMIT":   c.Clone.Sha,
+
+		"DRONE":        "true",
+		"DRONE_REPO":   c.Repo.FullName,
+		"DRONE_BUILD":  strconv.Itoa(c.Commit.Sequence),
+		"DRONE_BRANCH": c.Clone.Branch,
+		"DRONE_COMMIT": c.Clone.Sha,
+		"DRONE_DIR":    c.Clone.Dir,
+	}
+}
+
+// helper function to encode the build step to
+// a json string. Primarily used for plugins, which
+// expect a json encoded string in stdin or arg[1].
+func toCommand(c *Context, step *common.Step) []string {
+	p := payload{
+		c.Repo,
+		c.Commit,
+		c.Build,
+		c.Clone,
+		step.Config,
+	}
+	return []string{p.Encode()}
+}
+
+// payload represents the payload of a plugin
+// that is serialized and sent to the plugin in JSON
+// format via stdin or arg[1].
+type payload struct {
+	Repo   *common.Repo   `json:"repo"`
+	Commit *common.Commit `json:"commit"`
+	Build  *common.Build  `json:"build"`
+	Clone  *common.Clone  `json:"clone"`
+
+	Config map[string]interface{} `json:"vargs"`
+}
+
+// Encode encodes the payload in JSON format.
+func (p *payload) Encode() string {
+	out, _ := json.Marshal(p)
+	return string(out)
+}