starlark converter & parser created and working. Unit tests add & some moving around/refactoring of plugins.

cmd/drone-server/inject_plugin.go

@@ -20,7 +20,6 @@ import (
 	"github.com/drone/drone/plugin/admission"
 	"github.com/drone/drone/plugin/config"
 	"github.com/drone/drone/plugin/converter"
-	"github.com/drone/drone/plugin/converter/starlark"
 	"github.com/drone/drone/plugin/registry"
 	"github.com/drone/drone/plugin/secret"
 	"github.com/drone/drone/plugin/validator"
@@ -77,15 +76,18 @@ func provideConfigPlugin(client *scm.Client, contents core.FileService, conf spe
 // provideConvertPlugin is a Wire provider function that returns
 // a yaml conversion plugin based on the environment
 // configuration.
-func provideConvertPlugin(client *scm.Client, conf spec.Config) core.ConvertService {
+func provideConvertPlugin(client *scm.Client, conf spec.Config, templateStore core.TemplateStore) core.ConvertService {
 	return converter.Combine(
 		converter.Legacy(false),
-		starlark.New(
+		converter.New(
 			conf.Starlark.Enabled,
 		),
 		converter.Jsonnet(
 			conf.Jsonnet.Enabled,
 		),
+		converter.Template(
+			templateStore,
+		),
 		converter.Memoize(
 			converter.Remote(
 				conf.Convert.Endpoint,

cmd/drone-server/wire_gen.go

@@ -65,7 +65,8 @@ func InitializeApplication(config2 config.Config) (application, error) {
 	coreCanceler := canceler.New(buildStore, corePubsub, repositoryStore, scheduler, stageStore, statusService, stepStore, userStore, webhookSender)
 	fileService := provideContentService(client, renewer)
 	configService := provideConfigPlugin(client, fileService, config2)
-	convertService := provideConvertPlugin(client, config2)
+	templateStore := template.New(db)
+	convertService := provideConvertPlugin(client, config2, templateStore)
 	validateService := provideValidatePlugin(config2)
 	triggerer := trigger.New(coreCanceler, configService, convertService, commitService, statusService, buildStore, scheduler, repositoryStore, userStore, validateService, webhookSender)
 	cronScheduler := cron2.New(commitService, cronStore, repositoryStore, userStore, triggerer)
@@ -92,7 +93,6 @@ func InitializeApplication(config2 config.Config) (application, error) {
 	}
 	batcher := provideBatchStore(db, config2)
 	syncer := provideSyncer(repositoryService, repositoryStore, userStore, batcher, config2)
-	templateStore := template.New(db)
 	transferer := transfer.New(repositoryStore, permStore)
 	userService := user.New(client, renewer)
 	server := api.New(buildStore, commitService, cronStore, corePubsub, globalSecretStore, hookService, logStore, coreLicense, licenseService, organizationService, permStore, repositoryStore, repositoryService, scheduler, secretStore, stageStore, stepStore, statusService, session, logStream, syncer, system, templateStore, transferer, triggerer, userStore, userService, webhookSender)

handler/api/template/create.go

@@ -20,7 +20,7 @@ type templateInput struct {
 
 // HandleCreate returns an http.HandlerFunc that processes http
 // requests to create a new template.
-func HandleCreate(secrets core.TemplateStore) http.HandlerFunc {
+func HandleCreate(templateStore core.TemplateStore) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		in := new(templateInput)
 		err := json.NewDecoder(r.Body).Decode(in)
@@ -40,7 +40,7 @@ func HandleCreate(secrets core.TemplateStore) http.HandlerFunc {
 			return
 		}
 
-		err = secrets.Create(r.Context(), t)
+		err = templateStore.Create(r.Context(), t)
 		if err != nil {
 			render.InternalError(w, err)
 			return

handler/web/pages.go

@@ -87,7 +87,7 @@ func setupCache(h http.Handler) http.Handler {
 // 	string(dist.MustLookup("/index.html")),
 // )
 
-// // default func map with json parser.
+// // default func map with json parsers.
 // var funcMap = template.FuncMap{
 // 	"json": func(v interface{}) template.JS {
 // 		a, _ := json.Marshal(v)

plugin/converter/parser/args.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package starlark
+package parser
 
 import (
 	"github.com/drone/drone/core"
@@ -38,18 +38,29 @@ import (
 // TODO(bradrydzewski) add build parent
 // TODO(bradrydzewski) add build timestamp
 
-func createArgs(repo *core.Repository, build *core.Build) []starlark.Value {
+func createArgs(repo *core.Repository, build *core.Build, input map[string]interface{}) []starlark.Value {
 	return []starlark.Value{
 		starlarkstruct.FromStringDict(
 			starlark.String("context"),
 			starlark.StringDict{
 				"repo":  starlarkstruct.FromStringDict(starlark.String("repo"), fromRepo(repo)),
 				"build": starlarkstruct.FromStringDict(starlark.String("build"), fromBuild(build)),
+				"input": starlarkstruct.FromStringDict(starlark.String("input"), fromInput(input)),
 			},
 		),
 	}
 }
 
+func fromInput(input map[string]interface{}) starlark.StringDict {
+	out := map[string]starlark.Value{}
+	for k, v := range input {
+		if s, ok := v.(string); ok {
+			out[k] = starlark.String(s)
+		}
+	}
+	return out
+}
+
 func fromBuild(v *core.Build) starlark.StringDict {
 	return starlark.StringDict{
 		"event":         starlark.String(v.Event),

plugin/converter/parser/starlark.go

@@ -1,26 +1,9 @@
-// Copyright 2019 Drone IO, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package starlark
+package parser
 
 import (
 	"bytes"
-	"context"
-	"errors"
-	"strings"
-
 	"github.com/drone/drone/core"
+	"github.com/drone/drone/handler/api/errors"
 	"github.com/sirupsen/logrus"
 	"go.starlark.net/starlark"
 )
@@ -55,33 +38,7 @@ var (
 	ErrCannotLoad = errors.New("starlark: cannot load external scripts")
 )
 
-// New returns a conversion service that converts the
-// starlark file to a yaml file.
-func New(enabled bool) core.ConvertService {
-	return &starlarkPlugin{
-		enabled: enabled,
-	}
-}
-
-type starlarkPlugin struct {
-	enabled bool
-}
-
-func (p *starlarkPlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Config, error) {
-	if p.enabled == false {
-		return nil, nil
-	}
-
-	// if the file extension is not jsonnet we can
-	// skip this plugin by returning zero values.
-	switch {
-	case strings.HasSuffix(req.Repo.Config, ".script"):
-	case strings.HasSuffix(req.Repo.Config, ".star"):
-	case strings.HasSuffix(req.Repo.Config, ".starlark"):
-	default:
-		return nil, nil
-	}
-
+func ParseStarlark(req *core.ConvertArgs, template *core.Template, templateData map[string]interface{}) (file *string, err error) {
 	thread := &starlark.Thread{
 		Name: "drone",
 		Load: noLoad,
@@ -92,7 +49,17 @@ func (p *starlarkPlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*c
 			}).Traceln(msg)
 		},
 	}
-	globals, err := starlark.ExecFile(thread, req.Repo.Config, []byte(req.Config.Data), nil)
+	var starlarkFile []byte
+	var starlarkFileName string
+	if template != nil {
+		starlarkFile = template.Data
+		starlarkFileName = template.Name
+	} else {
+		starlarkFile = []byte(req.Config.Data)
+		starlarkFileName = req.Repo.Config
+	}
+
+	globals, err := starlark.ExecFile(thread, starlarkFileName, starlarkFile, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -111,7 +78,7 @@ func (p *starlarkPlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*c
 
 	// create the input args and invoke the main method
 	// using the input args.
-	args := createArgs(req.Repo, req.Build)
+	args := createArgs(req.Repo, req.Build, templateData)
 
 	// set the maximum number of operations in the script. this
 	// mitigates long running scripts.
@@ -146,12 +113,10 @@ func (p *starlarkPlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*c
 	// this is a temporary workaround until we
 	// implement a LimitWriter.
 	if b := buf.Bytes(); len(b) > limit {
-		return nil, ErrMaximumSize
+		return nil, nil
 	}
-
-	return &core.Config{
-		Data: buf.String(),
-	}, nil
+	parsedFile := buf.String()
+	return &parsedFile, nil
 }
 
 func noLoad(_ *starlark.Thread, _ string) (starlark.StringDict, error) {

plugin/converter/parser/starlark_test.go

@@ -0,0 +1,92 @@
+package parser
+
+import (
+	"github.com/drone/drone/core"
+	"io/ioutil"
+	"testing"
+)
+
+func TestParseStarlark(t *testing.T) {
+	before, err := ioutil.ReadFile("../testdata/starlark.input.star")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	after, err := ioutil.ReadFile("../testdata/starlark.input.star.golden")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.yml",
+		},
+		Config: &core.Config{},
+	}
+	template := &core.Template{
+		Name: "my_template.star",
+		Data: before,
+	}
+
+	templateData := map[string]interface{}{
+		"stepName": "my_step",
+		"image":    "my_image",
+		"commands": "my_command",
+	}
+
+	req.Config.Data = string(before)
+
+	parsedFile, err := ParseStarlark(req, template, templateData)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if want, got := *parsedFile, string(after); want != got {
+		t.Errorf("Want %q got %q", want, got)
+	}
+}
+
+func TestParseStarlarkNotTemplateFile(t *testing.T) {
+	before, err := ioutil.ReadFile("../testdata/single.star")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	after, err := ioutil.ReadFile("../testdata/single.star.golden")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.star",
+		},
+		Config: &core.Config{},
+	}
+
+	req.Repo.Config = "plugin.starlark.star"
+	req.Config.Data = string(before)
+
+	parsedFile, err := ParseStarlark(req, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if want, got := *parsedFile, string(after); want != got {
+		t.Errorf("Want %q got %q", want, got)
+	}
+}

plugin/converter/parser/write.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package starlark
+package parser
 
 import (
 	"encoding/json"

plugin/converter/starlark.go

@@ -0,0 +1,55 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package converter
+
+import (
+	"context"
+	"github.com/drone/drone/core"
+	"github.com/drone/drone/plugin/converter/parser"
+	"strings"
+)
+
+// New returns a conversion service that converts the
+// starlark file to a yaml file.
+func New(enabled bool) core.ConvertService {
+	return &starlarkPlugin{
+		enabled: enabled,
+	}
+}
+
+type starlarkPlugin struct {
+	enabled bool
+}
+
+func (p *starlarkPlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Config, error) {
+	if p.enabled == false {
+		return nil, nil
+	}
+
+	// if the file extension is not jsonnet we can
+	// skip this plugin by returning zero values.
+	switch {
+	case strings.HasSuffix(req.Repo.Config, ".script"):
+	case strings.HasSuffix(req.Repo.Config, ".star"):
+	case strings.HasSuffix(req.Repo.Config, ".starlark"):
+	default:
+		return nil, nil
+	}
+
+	file, _ := parser.ParseStarlark(req, nil, nil)
+	return &core.Config{
+		Data: *file,
+	}, nil
+}

plugin/converter/starlark_test.go

@@ -12,19 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package starlark
+package converter
 
 import (
-	"context"
 	"io/ioutil"
 	"testing"
 
 	"github.com/drone/drone/core"
 )
 
-var noContext = context.Background()
-
-func TestConvert(t *testing.T) {
+func TestStarlarkConvert(t *testing.T) {
 	plugin := New(true)
 
 	req := &core.ConvertArgs{

plugin/converter/template.go

@@ -0,0 +1,69 @@
+package converter
+
+import (
+	"context"
+	"errors"
+	"github.com/drone/drone/core"
+
+	"github.com/drone/drone/plugin/converter/parser"
+
+	"gopkg.in/yaml.v2"
+	"regexp"
+	"strings"
+)
+
+var (
+	// templateFileRE regex to verifying kind is template.
+	templateFileRE          = regexp.MustCompile("^kind:\\s+template+\\n")
+	ErrTemplateNotFound     = errors.New("template converter: template name given not found")
+	ErrTemplateSyntaxErrors = errors.New("template converter: there is a problem with the yaml file provided")
+)
+
+func TemplateConverter(templateStore core.TemplateStore) core.ConvertService {
+	return &templatePlugin{
+		templateStore: templateStore,
+	}
+}
+
+type templatePlugin struct {
+	templateStore core.TemplateStore
+}
+
+func (p *templatePlugin) Convert(ctx context.Context, req *core.ConvertArgs) (*core.Config, error) {
+	// check type is yaml
+	if strings.HasSuffix(req.Repo.Config, ".yml") == false {
+		return nil, nil
+	}
+	// check kind is template
+	if templateFileRE.MatchString(req.Config.Data) == false {
+		return nil, nil
+	}
+	// map to templateArgs
+	var templateArgs core.TemplateArgs
+	err := yaml.Unmarshal([]byte(req.Config.Data), &templateArgs)
+	if err != nil {
+		return nil, ErrTemplateSyntaxErrors
+	}
+	// get template from db
+	template, err := p.templateStore.FindName(ctx, templateArgs.Load)
+	if err != nil {
+		return nil, nil
+	}
+	if template == nil {
+		return nil, ErrTemplateNotFound
+	}
+	// Check if file is Starlark
+	if strings.HasSuffix(templateArgs.Load, ".script") ||
+		strings.HasSuffix(templateArgs.Load, ".star") ||
+		strings.HasSuffix(templateArgs.Load, ".starlark") {
+
+		file, err := parser.ParseStarlark(req, template, templateArgs.Data)
+		if err != nil {
+			return nil, err
+		}
+		return &core.Config{
+			Data: *file,
+		}, nil
+	}
+	return nil, nil
+}

plugin/converter/template_test.go

@@ -0,0 +1,157 @@
+package converter
+
+import (
+	"github.com/drone/drone/core"
+	"github.com/drone/drone/mock"
+	"github.com/golang/mock/gomock"
+	"io/ioutil"
+	"testing"
+)
+
+func TestTemplatePluginConvert(t *testing.T) {
+	beforeInput, err := ioutil.ReadFile("testdata/starlark.input.star")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	after, err := ioutil.ReadFile("testdata/starlark.input.star.golden")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	templateArgs, err := ioutil.ReadFile("testdata/starlark.template.yml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	template := &core.Template{
+		Name: "plugin.starlark",
+		Data: beforeInput,
+	}
+
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	templates := mock.NewMockTemplateStore(controller)
+	templates.EXPECT().FindName(gomock.Any(), template.Name).Return(template, nil)
+
+	plugin := TemplateConverter(templates)
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.yml",
+		},
+		Config: &core.Config{
+			Data: string(templateArgs),
+		},
+	}
+
+	config, err := plugin.Convert(noContext, req)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if config == nil {
+		t.Error("Want non-nil configuration")
+		return
+	}
+
+	if want, got := config.Data, string(after); want != got {
+		t.Errorf("Want %q got %q", want, got)
+	}
+}
+
+func TestTemplatePluginConvertNotYamlFile(t *testing.T) {
+
+	plugin := TemplateConverter(nil)
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.star",
+		},
+		Config: &core.Config{},
+	}
+
+	config, err := plugin.Convert(noContext, req)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if config != nil {
+		t.Errorf("Expect nil config returned for non-starlark files")
+	}
+}
+
+func TestTemplatePluginConvertDroneFileTypePipeline(t *testing.T) {
+	args, err := ioutil.ReadFile("testdata/drone.yml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	plugin := TemplateConverter(nil)
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.yml",
+		},
+		Config: &core.Config{Data: string(args)},
+	}
+
+	config, err := plugin.Convert(noContext, req)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if config != nil {
+		t.Errorf("Expect nil config returned for non-starlark files")
+	}
+}
+
+func TestTemplatePluginConvertTemplateNotFound(t *testing.T) {
+	controller := gomock.NewController(t)
+	defer controller.Finish()
+
+	template := &core.Template{
+		Name: "plugin.starlark",
+		Data: nil,
+	}
+
+	templates := mock.NewMockTemplateStore(controller)
+	templates.EXPECT().FindName(gomock.Any(), template.Name).Return(nil, nil)
+
+	templateArgs, err := ioutil.ReadFile("testdata/starlark.template.yml")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	plugin := TemplateConverter(templates)
+	req := &core.ConvertArgs{
+		Build: &core.Build{
+			After: "3d21ec53a331a6f037a91c368710b99387d012c1",
+		},
+		Repo: &core.Repository{
+			Slug:   "octocat/hello-world",
+			Config: ".drone.yml",
+		},
+		Config: &core.Config{Data: string(templateArgs)},
+	}
+
+	config, err := plugin.Convert(noContext, req)
+	if config != nil {
+		t.Errorf("template converter: template name given not found")
+	}
+}

plugin/converter/testdata/drone.yml

@@ -0,0 +1,6 @@
+kind: pipeline
+load: plugin.starlark
+data:
+  stepName: my_step
+  image: my_image
+  commands: my_command

plugin/converter/testdata/starlark.input.star

@@ -0,0 +1,14 @@
+def main(ctx):
+  return {
+    "kind": "pipeline",
+    "name": "build",
+    "steps": [
+      {
+        "name": ctx.input.stepName,
+        "image": ctx.input.image,
+        "commands": [
+            ctx.input.commands
+        ]
+      }
+    ]
+  }

plugin/converter/testdata/starlark.input.star.golden

@@ -0,0 +1 @@
+{"kind": "pipeline", "name": "build", "steps": [{"name": "my_step", "image": "my_image", "commands": ["my_command"]}]}

plugin/converter/testdata/starlark.template.yml

@@ -0,0 +1,6 @@
+kind: template
+load: plugin.starlark
+data:
+  stepName: my_step
+  image: my_image
+  commands: my_command

service/hook/parser/parse.go

@@ -84,7 +84,7 @@ func (p *parser) Parse(req *http.Request, secretFunc func(string) string) (*core
 		os.Stderr.Write(out)
 	}
 
-	// callback function provides the webhook parser with
+	// callback function provides the webhook parsers with
 	// a per-repository secret key used to verify the webhook
 	// payload signature for authenticity.
 	fn := func(webhook scm.Webhook) (string, error) {