From 1b8a00f5f696815140a5b79f01244cc6ada82563 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Thu, 20 Aug 2015 15:54:23 -0700 Subject: [PATCH] always check for new plugins, until we have an update strategy --- cmd/drone-build/client.go | 8 +++++ doc/build/README.md | 1 + doc/build/cache.md | 57 ++++++++++++++++++++++++++++++++++++ doc/build/env.md | 2 +- pkg/runner/builtin/worker.go | 5 ++++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 doc/build/cache.md diff --git a/cmd/drone-build/client.go b/cmd/drone-build/client.go index 02d4e413..3e4126ee 100644 --- a/cmd/drone-build/client.go +++ b/cmd/drone-build/client.go @@ -3,6 +3,7 @@ package main import ( "errors" "os" + "strings" log "github.com/drone/drone/Godeps/_workspace/src/github.com/Sirupsen/logrus" "github.com/drone/drone/Godeps/_workspace/src/github.com/samalba/dockerclient" @@ -184,6 +185,13 @@ func daemon(client dockerclient.Client, conf *dockerclient.ContainerConfig, pull client.PullImage(conf.Image, nil) } + // TEMPORARY: always try to pull the new image for now + // since we'll be frequently updating the plugin images + // over the next few weeks + if strings.HasPrefix(conf.Image, "plugins/") { + client.PullImage(conf.Image, nil) + } + // attempts to create the contianer id, err := client.CreateContainer(conf, "") if err != nil { diff --git a/doc/build/README.md b/doc/build/README.md index 72b8210f..d6ddfd0d 100644 --- a/doc/build/README.md +++ b/doc/build/README.md @@ -1,5 +1,6 @@ * [Overview](overview.md) * [Variables](env.md) +* [Cache](cache.md) * [Clone](clone.md) * [Build](build.md) * [Services](services.md) diff --git a/doc/build/cache.md b/doc/build/cache.md new file mode 100644 index 00000000..a3e0b175 --- /dev/null +++ b/doc/build/cache.md @@ -0,0 +1,57 @@ +# Caching +Drone allows the caching of directories within the workspace. This can be used to improve the performance of the builds. This document discusses how caching is implemented and provides some general advice for lowering the execution time of builds. + +Remember that cached volumes are retained between builds and those volumes are accessible through plugins. They are also available through the host's file system. Any information that is sensitive should not be contained within the cache. + +## Caching Implementation +Drone uses Docker volumes to handle caching. The volumes are mounted within the build container and are available immediately within the container and any plugins running within the environment. + +Given the following configuration. +```yaml +build: + cache: + - foo + - fizz/buzz +``` +Two volumes will be created with their paths resembling the following, with the path left of the : being the location on the host and to the right being the directory within the container. If a different path is specified in the [clone](clone.md) plugin then the paths on the right will be relative to the value in `path` rather than the default paths specified below. +``` +/tmp/drone/cache/${origin}/${org}/${name}/foo:/drone/src/${origin}/${org}/${name}/foo +/tmp/drone/cache/${origin}/${org}/${name}/fizz/buzz:/drone/src/${origin}/${org}/${name}/fizz/buzz +``` +If the drone repository used this caching then the volumes would look like. +``` +/tmp/drone/cache/github.com/drone/drone/foo:/drone/src/github.com/drone/drone/foo +/tmp/drone/cache/github.com/drone/drone/fizz/buzz:/drone/src/github.com/drone/drone/fizz/buzz +``` + +### Sharing the Cache +Volumes in Docker are local to the host. When using multiple build machines there will be a copy of the cache on each of these machines. If its desireable to share the cached directories among all the build machines then a distributed file system could be mounted to `/tmp/drone/cache`. Setting up a distrubted file system is beyond the scope of this document but can potentially benifit build times and reduce the disk usage from caching. + +### Deleting the Cache +The caching implementation uses the `/tmp/` directory which is typically cleared every boot unless explicitly changed. If the docker host is never rebooted then a cron job could be used to clear the `/tmp/drone/cache/` periodically. The amount of time to wait clear is dependent upon the number of builds being run, the amount of data being cached, and the size of the repositories. Because of this it is recommended to monitor the disk usage and then make a judgement call based on that data. + +## Optimization Strategies +The following sections, arranged alphabetically, contain optimization strategies for various programming languages and version control systems using caching. +### C/C++ +_Looking for additional input_ +### Dart +_Looking for additional input_ +### Git +When git is used as the version control system the .git directory is a good candidate for caching. Since the git plugin handles the checkout process the only thing needed is to add the .git directory to the cache section. +```yaml +build: + cache: + - .git +``` +### Golang +_Looking for additional input_ +### Java +_Looking for additional input_ +### JavaScript +_Looking for additional input_ +### PHP +_Looking for additional input_ +### Python +_Looking for additional input_ +### Ruby +_Looking for additional input_ diff --git a/doc/build/env.md b/doc/build/env.md index 43f34150..9b1a55ce 100644 --- a/doc/build/env.md +++ b/doc/build/env.md @@ -15,7 +15,7 @@ The build environment has access to the following environment variables: You may also store encrypted, private variables in the `.drone.yml` and inject at runtime. Private variables are encrypted using RSA encryption with OAEP (see [EncryptOAEP](http://golang.org/pkg/crypto/rsa/#EncryptOAEP)). You can generate encrypted strings from your repository settings screen. -Once you have an ecrypted string, you can add to the `secure` section of the `.drone.yml`.These variables are decrypted and injected into the `.drone.yml` at runtime using the `$$` notation. +Once you have an ecrypted string, you can add the encrypted variable to the `secure` section of the `.drone.yml`. These variables are decrypted and injected into the `.drone.yml` at runtime using the `$$` notation. An example `.drone.yml` expecting the `HEROKU_TOKEN` private variable: diff --git a/pkg/runner/builtin/worker.go b/pkg/runner/builtin/worker.go index c42e0a98..cc454b97 100644 --- a/pkg/runner/builtin/worker.go +++ b/pkg/runner/builtin/worker.go @@ -104,6 +104,11 @@ func (w *worker) Build(name string, stdin []byte, pr bool) (_ int, err error) { }, } + // TEMPORARY: always try to pull the new image for now + // since we'll be frequently updating the build image + // for the next few weeks + w.client.PullImage(conf.Image, nil) + w.build, err = run(w.client, conf, name, w.timeout) if err != nil { return 1, err