enable caching of remote config requests

cmd/drone-server/config/config.go

@@ -98,7 +98,7 @@ type (
 	}
 
 	Cleanup struct {
-		Disabled  bool         `envconfig:"DRONE_CLEANUP_DISABLED"`
+		Disabled bool          `envconfig:"DRONE_CLEANUP_DISABLED"`
 		Interval time.Duration `envconfig:"DRONE_CLEANUP_INTERVAL"         default:"24h"`
 		Running  time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_RUNNING" default:"24h"`
 		Pending  time.Duration `envconfig:"DRONE_CLEANUP_DEADLINE_PENDING" default:"24h"`
@@ -188,6 +188,10 @@ type (
 		Filter     []string `envconfig:"DRONE_REPOSITORY_FILTER"`
 		Visibility string   `envconfig:"DRONE_REPOSITORY_VISIBILITY"`
 		Trusted    bool     `envconfig:"DRONE_REPOSITORY_TRUSTED"`
+
+		// THIS SETTING IS INTERNAL USE ONLY AND SHOULD
+		// NOT BE USED OR RELIED UPON IN PRODUCTION.
+		Ignore []string `envconfig:"DRONE_REPOSITORY_IGNORE"`
 	}
 
 	// Registries provides the registry configuration.

cmd/drone-server/inject_plugin.go

@@ -61,10 +61,12 @@ func provideAdmissionPlugin(client *scm.Client, orgs core.OrganizationService, u
 // configuration.
 func provideConfigPlugin(client *scm.Client, contents core.FileService, conf spec.Config) core.ConfigService {
 	return config.Combine(
-		config.Global(
+		config.Memoize(
-			conf.Yaml.Endpoint,
+			config.Global(
-			conf.Yaml.Secret,
+				conf.Yaml.Endpoint,
-			conf.Yaml.SkipVerify,
+				conf.Yaml.Secret,
+				conf.Yaml.SkipVerify,
+			),
 		),
 		config.Repository(contents),
 	)
@@ -80,11 +82,13 @@ func provideConvertPlugin(client *scm.Client, conf spec.Config) core.ConvertServ
 		converter.Jsonnet(
 			conf.Jsonnet.Enabled,
 		),
-		converter.Remote(
+		converter.Memoize(
-			conf.Convert.Endpoint,
+			converter.Remote(
-			conf.Convert.Secret,
+				conf.Convert.Endpoint,
-			conf.Convert.Extension,
+				conf.Convert.Secret,
-			conf.Convert.SkipVerify,
+				conf.Convert.Extension,
+				conf.Convert.SkipVerify,
+			),
 		),
 	)
 }
@@ -130,6 +134,12 @@ func provideValidatePlugin(conf spec.Config) core.ValidateService {
 			conf.Validate.Secret,
 			conf.Validate.SkipVerify,
 		),
+		// THIS FEATURE IS INTERNAL USE ONLY AND SHOULD
+		// NOT BE USED OR RELIED UPON IN PRODUCTION.
+		validator.Filter(
+			nil,
+			conf.Repository.Ignore,
+		),
 	)
 }
 

plugin/config/memoize.go

@@ -0,0 +1,105 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !oss
+
+package config
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/drone/drone/core"
+
+	lru "github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
+)
+
+// cache key pattern used in the cache, comprised of the
+// repository slug and commit sha.
+const keyf = "%d|%s|%s|%s|%s|%s"
+
+// Memoize caches the conversion results for subsequent calls.
+// This micro-optimization is intended for multi-pipeline
+// projects that would otherwise covert the file for each
+// pipeline execution.
+func Memoize(base core.ConfigService) core.ConfigService {
+	// simple cache prevents the same yaml file from being
+	// requested multiple times in a short period.
+	cache, _ := lru.New(10)
+	return &memoize{base: base, cache: cache}
+}
+
+type memoize struct {
+	base  core.ConfigService
+	cache *lru.Cache
+}
+
+func (c *memoize) Find(ctx context.Context, req *core.ConfigArgs) (*core.Config, error) {
+	// this is a minor optimization that prevents caching if the
+	// base converter is a global config service and is disabled.
+	if global, ok := c.base.(*global); ok == true && global.client == nil {
+		return nil, nil
+	}
+
+	// generate the key used to cache the converted file.
+	key := fmt.Sprintf(keyf,
+		req.Repo.ID,
+		req.Build.Event,
+		req.Build.Action,
+		req.Build.Ref,
+		req.Build.After,
+		req.Repo.Config,
+	)
+
+	logger := logrus.WithField("repo", req.Repo.Slug).
+		WithField("build", req.Build.Event).
+		WithField("action", req.Build.Action).
+		WithField("ref", req.Build.Ref).
+		WithField("rev", req.Build.After).
+		WithField("config", req.Repo.Config)
+
+	logger.Trace("extension: configuration: check cache")
+
+	// check the cache for the file and return if exists.
+	cached, ok := c.cache.Get(key)
+	if ok {
+		logger.Trace("extension: configuration: cache hit")
+		return cached.(*core.Config), nil
+	}
+
+	logger.Trace("extension: configuration: cache miss")
+
+	// else find the configuration file.
+	config, err := c.base.Find(ctx, req)
+	if err != nil {
+		return nil, err
+	}
+
+	if config == nil {
+		return nil, nil
+	}
+	if config.Data == "" {
+		return nil, nil
+	}
+
+	// if the configuration file was retrieved
+	// it is temporarily cached. Note that we do
+	// not cache if the commit sha is empty (gogs).
+	if req.Build.After != "" {
+		c.cache.Add(key, config)
+	}
+
+	return config, nil
+}

plugin/config/memoize_oss.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build oss
+
+package config
+
+import (
+	"github.com/drone/drone/core"
+)
+
+// Memoize caches the conversion results for subsequent calls.
+// This micro-optimization is intended for multi-pipeline
+// projects that would otherwise covert the file for each
+// pipeline execution.
+func Memoize(base core.ConvertService) core.ConvertService {
+	return new(noop)
+}

plugin/config/memoize_test.go

@@ -0,0 +1,159 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Drone Non-Commercial License
+// that can be found in the LICENSE file.
+
+// +build !oss
+
+package config
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/drone/drone/core"
+	"github.com/drone/drone/mock"
+
+	"github.com/golang/mock/gomock"
+)
+
+func TestMemoize(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	conf := &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"}
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: conf,
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	_, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if got, want := service.cache.Len(), 1; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+
+	args.Config = nil // set to nil to prove we get the cached value
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != conf {
+		t.Errorf("Expect result from cache")
+	}
+
+	if got, want := service.cache.Len(), 1; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Tag(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{Ref: "refs/tags/v1.0.0"},
+		Repo:   &core.Repository{ID: 42},
+		Config: &core.Config{Data: "{kind: pipeline, type: docker, steps: []}"},
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != args.Config {
+		t.Errorf("Expect result from cache")
+	}
+}
+
+func TestMemoize_Empty(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: &core.Config{Data: ""}, // empty
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != nil {
+		t.Errorf("Expect nil response")
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Nil(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build:  &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:   &core.Repository{ID: 42},
+		Config: nil,
+	}
+
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(args.Config, nil)
+
+	service := Memoize(base).(*memoize)
+	res, err := service.Find(noContext, args)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if res != nil {
+		t.Errorf("Expect nil response")
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}
+
+func TestMemoize_Error(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	args := &core.ConfigArgs{
+		Build: &core.Build{After: "3950521325d4744760a96c18e3d0c67d86495af3"},
+		Repo:  &core.Repository{ID: 42},
+	}
+
+	want := errors.New("not found")
+	base := mock.NewMockConfigService(controller)
+	base.EXPECT().Find(gomock.Any(), gomock.Any()).Return(nil, want)
+
+	service := Memoize(base).(*memoize)
+	_, err := service.Find(noContext, args)
+	if err == nil {
+		t.Errorf("Expect error from base returned to caller")
+		return
+	}
+	if got, want := service.cache.Len(), 0; got != want {
+		t.Errorf("Expect %d items in cache, got %d", want, got)
+	}
+}

plugin/converter/memoize.go

@@ -23,11 +23,12 @@ import (
 	"github.com/drone/drone/core"
 
 	lru "github.com/hashicorp/golang-lru"
+	"github.com/sirupsen/logrus"
 )
 
 // cache key pattern used in the cache, comprised of the
 // repository slug and commit sha.
-const keyf = "%d/%s"
+const keyf = "%d|%s|%s|%s|%s|%s"
 
 // Memoize caches the conversion results for subsequent calls.
 // This micro-optimization is intended for multi-pipeline
@@ -36,7 +37,7 @@ const keyf = "%d/%s"
 func Memoize(base core.ConvertService) core.ConvertService {
 	// simple cache prevents the same yaml file from being
 	// requested multiple times in a short period.
-	cache, _ := lru.New(25)
+	cache, _ := lru.New(10)
 	return &memoize{base: base, cache: cache}
 }
 
@@ -53,21 +54,33 @@ func (c *memoize) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Con
 	}
 
 	// generate the key used to cache the converted file.
-	key := fmt.Sprintf(keyf, req.Repo.ID, req.Build.After)
+	key := fmt.Sprintf(keyf,
+		req.Repo.ID,
+		req.Build.Event,
+		req.Build.Action,
+		req.Build.Ref,
+		req.Build.After,
+		req.Repo.Config,
+	)
 
-	// some source control management systems (gogs) do not provide
+	logger := logrus.WithField("repo", req.Repo.Slug).
-	// the commit sha for tag webhooks. If no commit sha is available
+		WithField("build", req.Build.Event).
-	// the ref is used.
+		WithField("action", req.Build.Action).
-	if req.Build.After == "" {
+		WithField("ref", req.Build.Ref).
-		key = fmt.Sprintf(keyf, req.Repo.ID, req.Build.Ref)
+		WithField("rev", req.Build.After).
-	}
+		WithField("config", req.Repo.Config)
+
+	logger.Trace("extension: conversion: check cache")
 
 	// check the cache for the file and return if exists.
 	cached, ok := c.cache.Get(key)
 	if ok {
+		logger.Trace("extension: conversion: cache hit")
 		return cached.(*core.Config), nil
 	}
 
+	logger.Trace("extension: conversion: cache miss")
+
 	// else convert the configuration file.
 	config, err := c.base.Convert(ctx, req)
 	if err != nil {
@@ -82,7 +95,11 @@ func (c *memoize) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Con
 	}
 
 	// if the configuration file was converted
-	// it is temporarily cached.
+	// it is temporarily cached. Note that we do
-	c.cache.Add(key, config)
+	// not cache if the commit sha is empty (gogs).
+	if req.Build.After != "" {
+		c.cache.Add(key, config)
+	}
+
 	return config, nil
 }

plugin/validator/filter.go

@@ -0,0 +1,66 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validator
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/drone/drone/core"
+)
+
+// Filter returns a validation service that skips
+// pipelines that do not match the filter criteria.
+func Filter(include, exclude []string) core.ValidateService {
+	return &filter{
+		include: include,
+		exclude: exclude,
+	}
+}
+
+type filter struct {
+	include []string
+	exclude []string
+}
+
+func (f *filter) Validate(ctx context.Context, in *core.ValidateArgs) error {
+	if len(f.include) > 0 {
+		for _, pattern := range f.include {
+			ok, _ := filepath.Match(pattern, in.Repo.Slug)
+			if ok {
+				return nil
+			}
+		}
+
+		// if the include list is specified, and the
+		// repository does not match any patterns in
+		// the include list, it should be skipped.
+		return core.ErrValidatorSkip
+	}
+
+	if len(f.exclude) > 0 {
+		for _, pattern := range f.exclude {
+			ok, _ := filepath.Match(pattern, in.Repo.Slug)
+			if ok {
+				// if the exclude list is specified, and
+				// the repository matches a pattern in the
+				// exclude list, it should be skipped.
+				return core.ErrValidatorSkip
+			}
+		}
+	}
+
+	return nil
+}

plugin/validator/filter_test.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package validator
+
+import (
+	"testing"
+
+	"github.com/drone/drone/core"
+)
+
+func TestFilter_None(t *testing.T) {
+	f := Filter(nil, nil)
+	if err := f.Validate(noContext, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestFilter_Include(t *testing.T) {
+	args := &core.ValidateArgs{
+		Repo: &core.Repository{Slug: "octocat/hello-world"},
+	}
+
+	f := Filter([]string{"octocat/hello-world"}, nil)
+	if err := f.Validate(noContext, args); err != nil {
+		t.Error(err)
+	}
+
+	f = Filter([]string{"octocat/*"}, nil)
+	if err := f.Validate(noContext, args); err != nil {
+		t.Error(err)
+	}
+
+	f = Filter([]string{"spaceghost/*"}, nil)
+	if err := f.Validate(noContext, args); err != core.ErrValidatorSkip {
+		t.Errorf("Expect ErrValidatorSkip, got %s", err)
+	}
+}
+
+func TestFilter_Exclude(t *testing.T) {
+	args := &core.ValidateArgs{
+		Repo: &core.Repository{Slug: "octocat/hello-world"},
+	}
+
+	f := Filter(nil, []string{"octocat/hello-world"})
+	if err := f.Validate(noContext, args); err != core.ErrValidatorSkip {
+		t.Errorf("Expect ErrValidatorSkip, got %s", err)
+	}
+
+	f = Filter(nil, []string{"octocat/*"})
+	if err := f.Validate(noContext, args); err != core.ErrValidatorSkip {
+		t.Errorf("Expect ErrValidatorSkip, got %s", err)
+	}
+
+	f = Filter(nil, []string{"spaceghost/*"})
+	if err := f.Validate(noContext, args); err != nil {
+		t.Error(err)
+	}
+}