diff --git a/cmd/drone-server/config/config.go b/cmd/drone-server/config/config.go index 2d9bd444..53fa6480 100644 --- a/cmd/drone-server/config/config.go +++ b/cmd/drone-server/config/config.go @@ -98,7 +98,7 @@ type ( } Cleanup struct { - Disabled bool `envconfig:"DRONE_CLEANUP_DISABLED"` + Disabled bool `envconfig:"DRONE_CLEANUP_DISABLED"` Interval time.Duration `envconfig:"DRONE_CLEANUP_INTERVAL" default:"24h"` Running time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_RUNNING" default:"24h"` Pending time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_PENDING" default:"24h"` @@ -188,6 +188,10 @@ type ( Filter []string `envconfig:"DRONE_REPOSITORY_FILTER"` Visibility string `envconfig:"DRONE_REPOSITORY_VISIBILITY"` Trusted bool `envconfig:"DRONE_REPOSITORY_TRUSTED"` + + // THIS SETTING IS INTERNAL USE ONLY AND SHOULD + // NOT BE USED OR RELIED UPON IN PRODUCTION. + Ignore []string `envconfig:"DRONE_REPOSITORY_IGNORE"` } // Registries provides the registry configuration. diff --git a/cmd/drone-server/inject_plugin.go b/cmd/drone-server/inject_plugin.go index 8c2f2a04..b46376d0 100644 --- a/cmd/drone-server/inject_plugin.go +++ b/cmd/drone-server/inject_plugin.go @@ -61,10 +61,12 @@ func provideAdmissionPlugin(client *scm.Client, orgs core.OrganizationService, u // configuration. func provideConfigPlugin(client *scm.Client, contents core.FileService, conf spec.Config) core.ConfigService { return config.Combine( - config.Global( - conf.Yaml.Endpoint, - conf.Yaml.Secret, - conf.Yaml.SkipVerify, + config.Memoize( + config.Global( + conf.Yaml.Endpoint, + conf.Yaml.Secret, + conf.Yaml.SkipVerify, + ), ), config.Repository(contents), ) @@ -80,11 +82,13 @@ func provideConvertPlugin(client *scm.Client, conf spec.Config) core.ConvertServ converter.Jsonnet( conf.Jsonnet.Enabled, ), - converter.Remote( - conf.Convert.Endpoint, - conf.Convert.Secret, - conf.Convert.Extension, - conf.Convert.SkipVerify, + converter.Memoize( + converter.Remote( + conf.Convert.Endpoint, + conf.Convert.Secret, + conf.Convert.Extension, + conf.Convert.SkipVerify, + ), ), ) } @@ -130,6 +134,12 @@ func provideValidatePlugin(conf spec.Config) core.ValidateService { conf.Validate.Secret, conf.Validate.SkipVerify, ), + // THIS FEATURE IS INTERNAL USE ONLY AND SHOULD + // NOT BE USED OR RELIED UPON IN PRODUCTION. + validator.Filter( + nil, + conf.Repository.Ignore, + ), ) } diff --git a/plugin/config/memoize.go b/plugin/config/memoize.go new file mode 100644 index 00000000..13adaae4 --- /dev/null +++ b/plugin/config/memoize.go @@ -0,0 +1,105 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !oss + +package config + +import ( + "context" + "fmt" + + "github.com/drone/drone/core" + + lru "github.com/hashicorp/golang-lru" + "github.com/sirupsen/logrus" +) + +// cache key pattern used in the cache, comprised of the +// repository slug and commit sha. +const keyf = "%d|%s|%s|%s|%s|%s" + +// Memoize caches the conversion results for subsequent calls. +// This micro-optimization is intended for multi-pipeline +// projects that would otherwise covert the file for each +// pipeline execution. +func Memoize(base core.ConfigService) core.ConfigService { + // simple cache prevents the same yaml file from being + // requested multiple times in a short period. + cache, _ := lru.New(10) + return &memoize{base: base, cache: cache} +} + +type memoize struct { + base core.ConfigService + cache *lru.Cache +} + +func (c *memoize) Find(ctx context.Context, req *core.ConfigArgs) (*core.Config, error) { + // this is a minor optimization that prevents caching if the + // base converter is a global config service and is disabled. + if global, ok := c.base.(*global); ok == true && global.client == nil { + return nil, nil + } + + // generate the key used to cache the converted file. + key := fmt.Sprintf(keyf, + req.Repo.ID, + req.Build.Event, + req.Build.Action, + req.Build.Ref, + req.Build.After, + req.Repo.Config, + ) + + logger := logrus.WithField("repo", req.Repo.Slug). + WithField("build", req.Build.Event). + WithField("action", req.Build.Action). + WithField("ref", req.Build.Ref). + WithField("rev", req.Build.After). + WithField("config", req.Repo.Config) + + logger.Trace("extension: configuration: check cache") + + // check the cache for the file and return if exists. + cached, ok := c.cache.Get(key) + if ok { + logger.Trace("extension: configuration: cache hit") + return cached.(*core.Config), nil + } + + logger.Trace("extension: configuration: cache miss") + + // else find the configuration file. + config, err := c.base.Find(ctx, req) + if err != nil { + return nil, err + } + + if config == nil { + return nil, nil + } + if config.Data == "" { + return nil, nil + } + + // if the configuration file was retrieved + // it is temporarily cached. Note that we do + // not cache if the commit sha is empty (gogs). + if req.Build.After != "" { + c.cache.Add(key, config) + } + + return config, nil +} diff --git a/plugin/config/memoize_oss.go b/plugin/config/memoize_oss.go new file mode 100644 index 00000000..6cacf629 --- /dev/null +++ b/plugin/config/memoize_oss.go @@ -0,0 +1,29 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build oss + +package config + +import ( + "github.com/drone/drone/core" +) + +// Memoize caches the conversion results for subsequent calls. +// This micro-optimization is intended for multi-pipeline +// projects that would otherwise covert the file for each +// pipeline execution. +func Memoize(base core.ConvertService) core.ConvertService { + return new(noop) +} diff --git a/plugin/config/memoize_test.go b/plugin/config/memoize_test.go new file mode 100644 index 00000000..74975776 --- /dev/null +++ b/plugin/config/memoize_test.go @@ -0,0 +1,159 @@ +// Copyright 2019 Drone.IO Inc. All rights reserved. +// Use of this source code is governed by the Drone Non-Commercial License +// that can be found in the LICENSE file. + +// +build !oss + +package config + +import ( + "errors" + "testing" + + "github.com/drone/drone/core" + "github.com/drone/drone/mock" + + "github.com/golang/mock/gomock" +) + +func TestMemoize(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + conf := &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"} + args := &core.ConfigArgs{ + Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"}, + Repo: &core.Repository{ID: 42}, + Config: conf, + } + + base := mock.NewMockConfigService(controller) + base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil) + + service := Memoize(base).(*memoize) + _, err := service.Find(noContext, args) + if err != nil { + t.Error(err) + return + } + + if got, want := service.cache.Len(), 1; got != want { + t.Errorf("Expect %d items in cache, got %d", want, got) + } + + args.Config = nil // set to nil to prove we get the cached value + res, err := service.Find(noContext, args) + if err != nil { + t.Error(err) + return + } + if res != conf { + t.Errorf("Expect result from cache") + } + + if got, want := service.cache.Len(), 1; got != want { + t.Errorf("Expect %d items in cache, got %d", want, got) + } +} + +func TestMemoize_Tag(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + args := &core.ConfigArgs{ + Build: &core.Build{Ref: "refs/tags/v1.0.0"}, + Repo: &core.Repository{ID: 42}, + Config: &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"}, + } + + base := mock.NewMockConfigService(controller) + base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil) + + service := Memoize(base).(*memoize) + res, err := service.Find(noContext, args) + if err != nil { + t.Error(err) + return + } + if res != args.Config { + t.Errorf("Expect result from cache") + } +} + +func TestMemoize_Empty(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + args := &core.ConfigArgs{ + Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"}, + Repo: &core.Repository{ID: 42}, + Config: &core.Config{Data: ""}, // empty + } + + base := mock.NewMockConfigService(controller) + base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil) + + service := Memoize(base).(*memoize) + res, err := service.Find(noContext, args) + if err != nil { + t.Error(err) + return + } + if res != nil { + t.Errorf("Expect nil response") + } + if got, want := service.cache.Len(), 0; got != want { + t.Errorf("Expect %d items in cache, got %d", want, got) + } +} + +func TestMemoize_Nil(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + args := &core.ConfigArgs{ + Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"}, + Repo: &core.Repository{ID: 42}, + Config: nil, + } + + base := mock.NewMockConfigService(controller) + base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil) + + service := Memoize(base).(*memoize) + res, err := service.Find(noContext, args) + if err != nil { + t.Error(err) + return + } + if res != nil { + t.Errorf("Expect nil response") + } + if got, want := service.cache.Len(), 0; got != want { + t.Errorf("Expect %d items in cache, got %d", want, got) + } +} + +func TestMemoize_Error(t *testing.T) { + controller := gomock.NewController(t) + defer controller.Finish() + + args := &core.ConfigArgs{ + Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"}, + Repo: &core.Repository{ID: 42}, + } + + want := errors.New("not found") + base := mock.NewMockConfigService(controller) + base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(nil, want) + + service := Memoize(base).(*memoize) + _, err := service.Find(noContext, args) + if err == nil { + t.Errorf("Expect error from base returned to caller") + return + } + if got, want := service.cache.Len(), 0; got != want { + t.Errorf("Expect %d items in cache, got %d", want, got) + } +} diff --git a/plugin/converter/memoize.go b/plugin/converter/memoize.go index 6d0bcaf4..4a6a32cd 100644 --- a/plugin/converter/memoize.go +++ b/plugin/converter/memoize.go @@ -23,11 +23,12 @@ import ( "github.com/drone/drone/core" lru "github.com/hashicorp/golang-lru" + "github.com/sirupsen/logrus" ) // cache key pattern used in the cache, comprised of the // repository slug and commit sha. -const keyf = "%d/%s" +const keyf = "%d|%s|%s|%s|%s|%s" // Memoize caches the conversion results for subsequent calls. // This micro-optimization is intended for multi-pipeline @@ -36,7 +37,7 @@ const keyf = "%d/%s" func Memoize(base core.ConvertService) core.ConvertService { // simple cache prevents the same yaml file from being // requested multiple times in a short period. - cache, _ := lru.New(25) + cache, _ := lru.New(10) return &memoize{base: base, cache: cache} } @@ -53,21 +54,33 @@ func (c *memoize) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Con } // generate the key used to cache the converted file. - key := fmt.Sprintf(keyf, req.Repo.ID, req.Build.After) + key := fmt.Sprintf(keyf, + req.Repo.ID, + req.Build.Event, + req.Build.Action, + req.Build.Ref, + req.Build.After, + req.Repo.Config, + ) - // some source control management systems (gogs) do not provide - // the commit sha for tag webhooks. If no commit sha is available - // the ref is used. - if req.Build.After == "" { - key = fmt.Sprintf(keyf, req.Repo.ID, req.Build.Ref) - } + logger := logrus.WithField("repo", req.Repo.Slug). + WithField("build", req.Build.Event). + WithField("action", req.Build.Action). + WithField("ref", req.Build.Ref). + WithField("rev", req.Build.After). + WithField("config", req.Repo.Config) + + logger.Trace("extension: conversion: check cache") // check the cache for the file and return if exists. cached, ok := c.cache.Get(key) if ok { + logger.Trace("extension: conversion: cache hit") return cached.(*core.Config), nil } + logger.Trace("extension: conversion: cache miss") + // else convert the configuration file. config, err := c.base.Convert(ctx, req) if err != nil { @@ -82,7 +95,11 @@ func (c *memoize) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Con } // if the configuration file was converted - // it is temporarily cached. - c.cache.Add(key, config) + // it is temporarily cached. Note that we do + // not cache if the commit sha is empty (gogs). + if req.Build.After != "" { + c.cache.Add(key, config) + } + return config, nil } diff --git a/plugin/validator/filter.go b/plugin/validator/filter.go new file mode 100644 index 00000000..0a109051 --- /dev/null +++ b/plugin/validator/filter.go @@ -0,0 +1,66 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validator + +import ( + "context" + "path/filepath" + + "github.com/drone/drone/core" +) + +// Filter returns a validation service that skips +// pipelines that do not match the filter criteria. +func Filter(include, exclude []string) core.ValidateService { + return &filter{ + include: include, + exclude: exclude, + } +} + +type filter struct { + include []string + exclude []string +} + +func (f *filter) Validate(ctx context.Context, in *core.ValidateArgs) error { + if len(f.include) > 0 { + for _, pattern := range f.include { + ok, _ := filepath.Match(pattern, in.Repo.Slug) + if ok { + return nil + } + } + + // if the include list is specified, and the + // repository does not match any patterns in + // the include list, it should be skipped. + return core.ErrValidatorSkip + } + + if len(f.exclude) > 0 { + for _, pattern := range f.exclude { + ok, _ := filepath.Match(pattern, in.Repo.Slug) + if ok { + // if the exclude list is specified, and + // the repository matches a pattern in the + // exclude list, it should be skipped. + return core.ErrValidatorSkip + } + } + } + + return nil +} diff --git a/plugin/validator/filter_test.go b/plugin/validator/filter_test.go new file mode 100644 index 00000000..990cc9a0 --- /dev/null +++ b/plugin/validator/filter_test.go @@ -0,0 +1,70 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validator + +import ( + "testing" + + "github.com/drone/drone/core" +) + +func TestFilter_None(t *testing.T) { + f := Filter(nil, nil) + if err := f.Validate(noContext, nil); err != nil { + t.Error(err) + } +} + +func TestFilter_Include(t *testing.T) { + args := &core.ValidateArgs{ + Repo: &core.Repository{Slug: "octocat/hello-world"}, + } + + f := Filter([]string{"octocat/hello-world"}, nil) + if err := f.Validate(noContext, args); err != nil { + t.Error(err) + } + + f = Filter([]string{"octocat/*"}, nil) + if err := f.Validate(noContext, args); err != nil { + t.Error(err) + } + + f = Filter([]string{"spaceghost/*"}, nil) + if err := f.Validate(noContext, args); err != core.ErrValidatorSkip { + t.Errorf("Expect ErrValidatorSkip, got %s", err) + } +} + +func TestFilter_Exclude(t *testing.T) { + args := &core.ValidateArgs{ + Repo: &core.Repository{Slug: "octocat/hello-world"}, + } + + f := Filter(nil, []string{"octocat/hello-world"}) + if err := f.Validate(noContext, args); err != core.ErrValidatorSkip { + t.Errorf("Expect ErrValidatorSkip, got %s", err) + } + + f = Filter(nil, []string{"octocat/*"}) + if err := f.Validate(noContext, args); err != core.ErrValidatorSkip { + t.Errorf("Expect ErrValidatorSkip, got %s", err) + } + + f = Filter(nil, []string{"spaceghost/*"}) + if err := f.Validate(noContext, args); err != nil { + t.Error(err) + } +}