From 42896b3711419683197a25b4f6abb0bc1477a4fa Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Thu, 28 May 2015 01:18:46 -0600 Subject: [PATCH 1/4] Replace pkg/settings with pkg/config --- cmd/drone-server/drone.go | 10 +++++----- pkg/config/config.go | 6 +++++- pkg/remote/github/github.go | 18 +++++++++--------- pkg/server/login.go | 22 +++++++++++----------- pkg/server/server.go | 14 +++++++------- pkg/server/session/session.go | 6 +++--- pkg/server/ws.go | 2 +- 7 files changed, 41 insertions(+), 37 deletions(-) diff --git a/cmd/drone-server/drone.go b/cmd/drone-server/drone.go index bb34b802..8f80ea6f 100644 --- a/cmd/drone-server/drone.go +++ b/cmd/drone-server/drone.go @@ -8,10 +8,10 @@ import ( "github.com/drone/drone/Godeps/_workspace/src/github.com/gin-gonic/gin" "github.com/drone/drone/Godeps/_workspace/src/github.com/elazarl/go-bindata-assetfs" + "github.com/drone/drone/pkg/config" "github.com/drone/drone/pkg/remote/github" "github.com/drone/drone/pkg/server" "github.com/drone/drone/pkg/server/session" - "github.com/drone/drone/pkg/settings" log "github.com/drone/drone/Godeps/_workspace/src/github.com/Sirupsen/logrus" eventbus "github.com/drone/drone/pkg/bus/builtin" @@ -37,7 +37,7 @@ var ( func main() { flag.Parse() - settings, err := settings.Parse(*conf) + settings, err := config.Load(*conf) if err != nil { panic(err) } @@ -46,8 +46,8 @@ func main() { store := store.New(db) defer db.Close() - remote := github.New(settings.Service) - session := session.New(settings.Session) + remote := github.New(settings) + session := session.New(settings) eventbus_ := eventbus.New() queue_ := queue.New() updater := runner.NewUpdater(eventbus_, store, remote) @@ -55,7 +55,7 @@ func main() { // launch the local queue runner if the system // is not conifugred to run in agent mode - if settings.Agents != nil && settings.Agents.Secret != "" { + if len(settings.Agents.Secret) != 0 { log.Infof("Run builds using remote build agents") } else { log.Infof("Run builds using the embedded build runner") diff --git a/pkg/config/config.go b/pkg/config/config.go index b6b93869..074b9c6d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -40,6 +40,10 @@ type Config struct { Expires int64 `envconfig:"optional"` } + Agents struct { + Secret string `envconfig:"optional"` + } + Database struct { Driver string `envconfig:"optional"` Datasource string `envconfig:"optional"` @@ -56,7 +60,7 @@ type Config struct { // Plugins represents a white-list of plugins // that the system is authorized to load. - Plugins []string `envconfig:"white_list"` + Plugins []string `envconfig:"optional"` } // Load loads the configuration file and reads diff --git a/pkg/remote/github/github.go b/pkg/remote/github/github.go index 8374f759..c10728be 100644 --- a/pkg/remote/github/github.go +++ b/pkg/remote/github/github.go @@ -10,7 +10,7 @@ import ( "time" "github.com/drone/drone/Godeps/_workspace/src/github.com/hashicorp/golang-lru" - "github.com/drone/drone/pkg/settings" + "github.com/drone/drone/pkg/config" common "github.com/drone/drone/pkg/types" "github.com/drone/drone/Godeps/_workspace/src/github.com/google/go-github/github" @@ -33,14 +33,14 @@ type GitHub struct { cache *lru.Cache } -func New(service *settings.Service) *GitHub { +func New(conf *config.Config) *GitHub { var github = GitHub{ API: DefaultAPI, URL: DefaultURL, - Client: service.OAuth.Client, - Secret: service.OAuth.Secret, - PrivateMode: service.PrivateMode, - SkipVerify: service.SkipVerify, + Client: conf.Auth.Client, + Secret: conf.Auth.Secret, + PrivateMode: conf.Remote.Private, + SkipVerify: conf.Remote.SkipVerify, } var err error github.cache, err = lru.New(1028) @@ -50,9 +50,9 @@ func New(service *settings.Service) *GitHub { // if GitHub enterprise then ensure we're using the // appropriate URLs - if !strings.HasPrefix(service.Base, DefaultURL) && len(service.Base) != 0 { - github.URL = service.Base - github.API = service.Base + "/api/v3/" + if !strings.HasPrefix(conf.Remote.Base, DefaultURL) && len(conf.Remote.Base) != 0 { + github.URL = conf.Remote.Base + github.API = conf.Remote.Base + "/api/v3/" } // the API must have a trailing slash if !strings.HasSuffix(github.API, "/") { diff --git a/pkg/server/login.go b/pkg/server/login.go index 002fe69b..b38a33a4 100644 --- a/pkg/server/login.go +++ b/pkg/server/login.go @@ -36,9 +36,9 @@ func GetLogin(c *gin.Context) { // Auth (username and password). This will delegate // authorization accordingly. switch { - case settings.Service.OAuth == nil: - getLoginBasic(c) - case settings.Service.OAuth.RequestToken != "": + // case settings.Auth == nil: + // getLoginBasic(c) + case settings.Auth.RequestToken != "": getLoginOauth1(c) default: getLoginOauth2(c) @@ -52,9 +52,9 @@ func GetLogin(c *gin.Context) { login := ToUser(c) // check organization membership, if applicable - if len(settings.Service.Orgs) != 0 { + if len(settings.Remote.Orgs) != 0 { orgs, _ := remote.Orgs(login) - if !checkMembership(orgs, settings.Service.Orgs) { + if !checkMembership(orgs, settings.Remote.Orgs) { c.Redirect(303, "/login#error=access_denied_org") return } @@ -73,7 +73,7 @@ func GetLogin(c *gin.Context) { // if self-registration is disabled we should // return a notAuthorized error. the only exception // is if no users exist yet in the system we'll proceed. - if !settings.Service.Open && count != 0 { + if !settings.Remote.Open && count != 0 { log.Errorf("cannot register %s. registration closed", login.Login) c.Redirect(303, "/login#error=access_denied") return @@ -137,11 +137,11 @@ func getLoginOauth2(c *gin.Context) { var remote = ToRemote(c) var config = &oauth2.Config{ - ClientId: settings.Service.OAuth.Client, - ClientSecret: settings.Service.OAuth.Secret, - Scope: strings.Join(settings.Service.OAuth.Scope, ","), - AuthURL: settings.Service.OAuth.Authorize, - TokenURL: settings.Service.OAuth.AccessToken, + ClientId: settings.Auth.Client, + ClientSecret: settings.Auth.Secret, + Scope: strings.Join(settings.Auth.Scope, ","), + AuthURL: settings.Auth.Authorize, + TokenURL: settings.Auth.AccessToken, RedirectURL: fmt.Sprintf("%s/authorize", httputil.GetURL(c.Request)), //settings.Server.Scheme, settings.Server.Hostname), } diff --git a/pkg/server/server.go b/pkg/server/server.go index 8df4b579..3ae4e87b 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -7,11 +7,11 @@ import ( "github.com/drone/drone/Godeps/_workspace/src/github.com/gin-gonic/gin" "github.com/drone/drone/pkg/bus" + "github.com/drone/drone/pkg/config" "github.com/drone/drone/pkg/queue" "github.com/drone/drone/pkg/remote" "github.com/drone/drone/pkg/runner" "github.com/drone/drone/pkg/server/session" - "github.com/drone/drone/pkg/settings" "github.com/drone/drone/pkg/store" common "github.com/drone/drone/pkg/types" ) @@ -91,17 +91,17 @@ func SetUpdater(u runner.Updater) gin.HandlerFunc { } } -func ToSettings(c *gin.Context) *settings.Settings { - v, ok := c.Get("settings") +func ToSettings(c *gin.Context) *config.Config { + v, ok := c.Get("config") if !ok { return nil } - return v.(*settings.Settings) + return v.(*config.Config) } -func SetSettings(s *settings.Settings) gin.HandlerFunc { +func SetSettings(s *config.Config) gin.HandlerFunc { return func(c *gin.Context) { - c.Set("settings", s) + c.Set("config", s) c.Next() } } @@ -249,7 +249,7 @@ func MustAgent() gin.HandlerFunc { conf := ToSettings(c) // verify remote agents are enabled - if conf.Agents == nil || len(conf.Agents.Secret) == 0 { + if len(conf.Agents.Secret) == 0 { c.AbortWithStatus(405) return } diff --git a/pkg/server/session/session.go b/pkg/server/session/session.go index 1182bb18..686c9fbf 100644 --- a/pkg/server/session/session.go +++ b/pkg/server/session/session.go @@ -6,7 +6,7 @@ import ( "time" "github.com/drone/drone/Godeps/_workspace/src/github.com/dgrijalva/jwt-go" - "github.com/drone/drone/pkg/settings" + "github.com/drone/drone/pkg/config" common "github.com/drone/drone/pkg/types" ) @@ -20,8 +20,8 @@ type session struct { expire time.Duration } -func New(s *settings.Session) Session { - secret := []byte(s.Secret) +func New(s *config.Config) Session { + secret := []byte(s.Session.Secret) expire := time.Hour * 72 return &session{ secret: secret, diff --git a/pkg/server/ws.go b/pkg/server/ws.go index a56d1a35..55adde1c 100644 --- a/pkg/server/ws.go +++ b/pkg/server/ws.go @@ -76,7 +76,7 @@ func GetStream(c *gin.Context) { // if the commit is being executed by an agent // we'll proxy the build output directly to the // remote Docker client, through the agent. - if conf.Agents != nil && conf.Agents.Secret != "" { + if conf.Agents.Secret != "" { addr, err := store.Agent(commit) if err != nil { c.Fail(500, err) From 145d88f39bfba239650c66368a9ce099bc7ab56f Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Thu, 28 May 2015 11:04:50 -0600 Subject: [PATCH 2/4] Fix tests, update README.md --- README.md | 26 +++++++++++++++++++------- pkg/server/token_test.go | 16 +++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 78ca0403..bc6812fb 100644 --- a/README.md +++ b/README.md @@ -31,20 +31,18 @@ path = "/etc/drone/drone.db" [docker] cert = "" key = "" -nodes = [ - "unix:///var/run/docker.sock", - "unix:///var/run/docker.sock" -] +addr = "unix:///var/run/docker.sock" +swarm = "" [service] -name = "github" +kind = "github" base = "https://github.com" orgs = [] open = false -private_mode = false +private = false skip_verify = true -[service.oauth] +[auth] client = "" secret = "" authorize = "https://github.com/login/oauth/authorize" @@ -54,3 +52,17 @@ request_token = "" [agents] secret = "" ``` + +Configuration settings can also be set by environment variables using the scheme `DRONE_
_`, substituting the section title for `
` and the key for ``, in all caps. For example: + +```shell +#!/bin/bash +# prepare environment for executing drone +DRONE_DOCKER_ADDR="tcp://10.0.0.1:2375" # for [docker] section, 'addr' setting +DRONE_AUTH_CLIENT="0123456789abcdef0123AA" # for [auth] section, 'client' setting +DRONE_AUTH_SECRET="" # for [auth] section, 'secret' setting + +exec bin/drone -config=drone.toml +``` + +_NOTE: Configuration settings from environment variables override values set in the TOML file._ diff --git a/pkg/server/token_test.go b/pkg/server/token_test.go index 628e0b4d..9fb7c537 100644 --- a/pkg/server/token_test.go +++ b/pkg/server/token_test.go @@ -12,9 +12,9 @@ import ( . "github.com/drone/drone/Godeps/_workspace/src/github.com/franela/goblin" "github.com/drone/drone/Godeps/_workspace/src/github.com/gin-gonic/gin" "github.com/drone/drone/Godeps/_workspace/src/github.com/stretchr/testify/mock" + "github.com/drone/drone/pkg/config" "github.com/drone/drone/pkg/server/recorder" "github.com/drone/drone/pkg/server/session" - "github.com/drone/drone/pkg/settings" "github.com/drone/drone/pkg/store/mock" "github.com/drone/drone/pkg/types" ) @@ -60,9 +60,10 @@ func TestToken(t *testing.T) { ctx.Set("datastore", store) ctx.Set("user", &types.User{Login: "Freya"}) - config := settings.Settings{Session: &settings.Session{Secret: "Otto"}} - ctx.Set("settings", &config) - ctx.Set("session", session.New(config.Session)) + conf := &config.Config{} + conf.Session.Secret = "Otto" + ctx.Set("settings", conf) + ctx.Set("session", session.New(conf)) // prepare the mock store.On("AddToken", mock.AnythingOfType("*types.Token")).Return(test.storeErr).Once() @@ -98,9 +99,10 @@ func TestToken(t *testing.T) { ctx.Set("datastore", store) ctx.Set("user", &types.User{Login: "Freya"}) - config := settings.Settings{Session: &settings.Session{Secret: "Otto"}} - ctx.Set("settings", &config) - ctx.Set("session", session.New(config.Session)) + conf := &config.Config{} + conf.Session.Secret = "Otto" + ctx.Set("settings", conf) + ctx.Set("session", session.New(conf)) // prepare the mock store.On("TokenLabel", mock.AnythingOfType("*types.User"), test.inLabel).Return(test.outToken, test.errTokenLabel).Once() From 8062df676639f13680303e89ecf81cb8c7ab6e82 Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Thu, 28 May 2015 11:56:15 -0600 Subject: [PATCH 3/4] Remove unused pkg/settings --- pkg/settings/settings.go | 137 --------------------------------------- 1 file changed, 137 deletions(-) delete mode 100644 pkg/settings/settings.go diff --git a/pkg/settings/settings.go b/pkg/settings/settings.go deleted file mode 100644 index 2af5abc7..00000000 --- a/pkg/settings/settings.go +++ /dev/null @@ -1,137 +0,0 @@ -package settings - -import "github.com/drone/drone/Godeps/_workspace/src/github.com/BurntSushi/toml" - -// Service represents the configuration details required -// to connect to the revision control system (ie GitHub, Bitbucket) -type Service struct { - // Name defines the name of the plugin. Possible values - // may be github, gitlab, bitbucket, or gogs. - Name string `toml:"name"` - - // Address defines the address (uri) of the plugin for - // communication via the net/rpc package. - Address string `toml:"address"` - - // Base defines the base URL for the service. For example: - // https://github.com - // https://bitbucket.org - // https://gitlab.drone.io - Base string `toml:"base"` - - // Indicates registration is open. If true any user - // will be able to setup an account. If false, the - // system administrator will need to provision accounts. - Open bool `toml:"open"` - - // Orgs defines a list of organizations the user - // must belong to in order to register. This will - // take precedence over the `Open` paramter. - Orgs []string `toml:"orgs"` - - // PrivateMode should be set to true if the - // remote system requires authentication for - // cloning public (open source) repositories. - PrivateMode bool `toml:"private_mode"` - - // SkipVerify instructs the client to skip SSL verification. - // This may be used with self-signed certificates, however, - // is not recommended for security reasons. - SkipVerify bool `toml:"skip_verify"` - - // OAuth configuration data. If nil or empty, Drone may - // assume basic authentication via username and password. - OAuth *OAuth `toml:"oauth"` -} - -// OAuth defines how a user should autheticate with the service. -// This supports OAuth2 and OAuth1 protocols. -type OAuth struct { - Client string `toml:"client"` - Secret string `toml:"secret"` - Authorize string `toml:"authorize"` - AccessToken string `toml:"access_token"` - RequestToken string `toml:"request_token"` - Scope []string `toml:"scope"` -} - -// Server represents the web server configuration details -// used to server HTTP requests. -type Server struct { - Base string `toml:"base"` - Addr string `toml:"addr"` - Cert string `toml:"cert"` - Key string `toml:"key"` - - Scheme string `toml:"scheme"` - Hostname string `toml:"hostname"` -} - -// Session represents the session configuration details -// used to generate, validate and expire authentication -// sessions. -type Session struct { - Secret string `toml:"secret"` - Expires int64 `toml:"expires"` -} - -// Docker represents the configuration details used -// to connect to the Docker daemon when scheduling -// and executing builds in containers. -type Docker struct { - Cert string `toml:"cert"` - Key string `toml:"key"` - Nodes []string `toml:"nodes"` -} - -// Database represents the configuration details used -// to connect to the embedded Bolt database. -type Database struct { - Driver string `toml:"driver"` - Datasource string `toml:"datasource"` -} - -type Agents struct { - Secret string `toml:"secret"` -} - -// Settings defines global settings for the Drone system. -type Settings struct { - Database *Database `toml:"database"` - Docker *Docker `toml:"docker"` - Service *Service `toml:"service"` - Server *Server `toml:"server"` - Session *Session `toml:"session"` - Agents *Agents `toml:"agents"` - - Plugins map[string]interface{} `toml:"plugins"` -} - -// Parse parses the Drone settings file at the specified path -// and unmarshals to a Settings structure. -func Parse(path string) (*Settings, error) { - s := &Settings{} - _, err := toml.DecodeFile(path, s) - return applyDefaults(s), err -} - -// ParseString parses the Drone settings string and unmarshals -// to a Settings structure. -func ParseString(data string) (*Settings, error) { - s := &Settings{} - _, err := toml.Decode(data, s) - return applyDefaults(s), err -} - -func applyDefaults(s *Settings) *Settings { - if s.Session == nil { - s.Session = &Session{} - } - // if no session token is provided we can - // instead use the client secret to sign - // our sessions and tokens. - if len(s.Session.Secret) == 0 { - s.Session.Secret = s.Service.OAuth.Secret - } - return s -} From 33666d71a01e52576f20a1ad200b017dc0aa32eb Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Fri, 29 May 2015 23:04:41 -0600 Subject: [PATCH 4/4] Finish removal of `pkg/settings` --- README.md | 2 +- pkg/config/config.go | 14 ++++++++++++-- pkg/remote/client/client.go | 6 +++--- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bc6812fb..ac7e62eb 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ DRONE_DOCKER_ADDR="tcp://10.0.0.1:2375" # for [docker] section, 'addr' setti DRONE_AUTH_CLIENT="0123456789abcdef0123AA" # for [auth] section, 'client' setting DRONE_AUTH_SECRET="" # for [auth] section, 'secret' setting -exec bin/drone -config=drone.toml +exec ./drone -config=drone.toml ``` _NOTE: Configuration settings from environment variables override values set in the TOML file._ diff --git a/pkg/config/config.go b/pkg/config/config.go index 074b9c6d..8f7d9634 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -66,7 +66,7 @@ type Config struct { // Load loads the configuration file and reads // parameters from environment variables. func Load(path string) (*Config, error) { - data, err := ioutil.ReadFile("drone.toml") + data, err := ioutil.ReadFile(path) if err != nil { return nil, err } @@ -85,5 +85,15 @@ func LoadBytes(data []byte) (*Config, error) { if err != nil { return nil, err } - return conf, nil + return applyDefaults(conf), nil +} + +func applyDefaults(c *Config) *Config { + // if no session token is provided we can + // instead use the client secret to sign + // our sessions and tokens. + if len(c.Session.Secret) == 0 { + c.Session.Secret = c.Auth.Secret + } + return c } diff --git a/pkg/remote/client/client.go b/pkg/remote/client/client.go index 638580b9..392c9783 100644 --- a/pkg/remote/client/client.go +++ b/pkg/remote/client/client.go @@ -5,7 +5,7 @@ import ( "net/http" "net/rpc" - "github.com/drone/drone/pkg/settings" + "github.com/drone/drone/pkg/config" common "github.com/drone/drone/pkg/types" ) @@ -17,8 +17,8 @@ type Client struct { // New returns a new, remote datastore backend that connects // via tcp and exchanges data using Go's RPC mechanism. -func New(service *settings.Service) (*Client, error) { - conn, err := net.Dial("tcp", service.Address) +func New(conf *config.Config) (*Client, error) { + conn, err := net.Dial("tcp", conf.Server.Addr) if err != nil { return nil, err }