Merge pull request #3072 from tphoney/add_steps_fields

(feat) adding depends_on, image and detached fields to step

core/step.go

@@ -30,6 +30,9 @@ type (
 		Started   int64    `json:"started,omitempty"`
 		Stopped   int64    `json:"stopped,omitempty"`
 		Version   int64    `json:"version"`
+		DependsOn []string `json:"depends_on,omitempty"`
+		Image     string   `json:"image,omitempty"`
+		Detached  bool     `json:"detached,omitempty"`
 	}
 
 	// StepStore persists build step information to storage.

store/shared/migrate/README.md

@@ -0,0 +1,32 @@
+# Building SQL DDL into Drone
+
+These folders contain the code for the different of databases that drone can use. They contain the SQL necessary to create the necessary tables and migrate between versions (IE the DDL). This SQL is generated into a go file and included as part of the Drone binary.
+
+## Making a changes to the database DDL
+
+Any new changes to the database structure are always put into a new SQL file. Follow the naming scheme in the `store/shared/migrate/<db>/files` of the SQL files by incrementing the number file name and give it a good description of what changes are being made.
+
+Changes will need to be implemented for all supported databases, making similar changes for eg Mysql/Postgres/Sqllite.
+
+**NB** Any changes to the database structure will need to be reflected for the relevant `struct` in the `core` directory. Changing the objects in the `store` directory for the ORM. Finally Possibly in the repositories github.com/drone/drone-go and  github.com/drone/runner-go.
+
+## Generating Go from the SQL files
+
+To generate the go files you will need to install the golang command line tool `Togo` so it is on your users PATH.
+
+### Steps to install Togo
+
+``` bash
+# in your workspace
+git clone git@github.com:bradrydzewski/togo.git
+cd togo
+go get github.com/bradrydzewski/togo
+```
+
+### Generating go DDL
+
+Enter the desired database's implementation folder, and run the following. It will update the `ddl_gen.go` file.
+
+``` bash
+go generate
+```

store/shared/migrate/mysql/ddl_gen.go

@@ -164,6 +164,18 @@ var migrations = []struct {
 		name: "create-index-template-namespace",
 		stmt: createIndexTemplateNamespace,
 	},
+	{
+		name: "alter-table-steps-add-column-step-depends-on",
+		stmt: alterTableStepsAddColumnStepDependsOn,
+	},
+	{
+		name: "alter-table-steps-add-column-step-image",
+		stmt: alterTableStepsAddColumnStepImage,
+	},
+	{
+		name: "alter-table-steps-add-column-step-detached",
+		stmt: alterTableStepsAddColumnStepDetached,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -686,3 +698,19 @@ CREATE TABLE IF NOT EXISTS templates (
 var createIndexTemplateNamespace = `
 CREATE INDEX ix_template_namespace ON templates (template_namespace);
 `
+
+//
+// 016_add_columns_steps.sql
+//
+
+var alterTableStepsAddColumnStepDependsOn = `
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NULL;
+`
+
+var alterTableStepsAddColumnStepImage = `
+ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT '';
+`
+
+var alterTableStepsAddColumnStepDetached = `
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;
+`

store/shared/migrate/mysql/files/016_add_columns_steps.sql

@@ -0,0 +1,11 @@
+-- name: alter-table-steps-add-column-step-depends-on
+
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NULL;
+
+-- name: alter-table-steps-add-column-step-image
+
+ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT '';
+
+-- name: alter-table-steps-add-column-step-detached
+
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;

store/shared/migrate/postgres/ddl_gen.go

@@ -156,6 +156,18 @@ var migrations = []struct {
 		name: "create-table-template",
 		stmt: createTableTemplate,
 	},
+	{
+		name: "alter-table-steps-add-column-step-depends-on",
+		stmt: alterTableStepsAddColumnStepDependsOn,
+	},
+	{
+		name: "alter-table-steps-add-column-step-image",
+		stmt: alterTableStepsAddColumnStepImage,
+	},
+	{
+		name: "alter-table-steps-add-column-step-detached",
+		stmt: alterTableStepsAddColumnStepDetached,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -658,3 +670,19 @@ CREATE TABLE IF NOT EXISTS templates (
 
 CREATE INDEX IF NOT EXISTS ix_template_namespace ON templates (template_namespace);
 `
+
+//
+// 017_add_columns_steps.sql
+//
+
+var alterTableStepsAddColumnStepDependsOn = `
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT '';
+`
+
+var alterTableStepsAddColumnStepImage = `
+ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT '';
+`
+
+var alterTableStepsAddColumnStepDetached = `
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;
+`

store/shared/migrate/postgres/files/017_add_columns_steps.sql

@@ -0,0 +1,11 @@
+-- name: alter-table-steps-add-column-step-depends-on
+
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT '';
+
+-- name: alter-table-steps-add-column-step-image
+
+ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT '';
+
+-- name: alter-table-steps-add-column-step-detached
+
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;

store/shared/migrate/sqlite/ddl_gen.go

@@ -156,6 +156,18 @@ var migrations = []struct {
 		name: "create-table-templates",
 		stmt: createTableTemplates,
 	},
+	{
+		name: "alter-table-steps-add-column-step-depends-on",
+		stmt: alterTableStepsAddColumnStepDependsOn,
+	},
+	{
+		name: "alter-table-steps-add-column-step-image",
+		stmt: alterTableStepsAddColumnStepImage,
+	},
+	{
+		name: "alter-table-steps-add-column-step-detached",
+		stmt: alterTableStepsAddColumnStepDetached,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -660,3 +672,19 @@ CREATE TABLE IF NOT EXISTS templates (
 
 CREATE INDEX IF NOT EXISTS ix_template_namespace ON templates (template_namespace);
 `
+
+//
+// 016_add_columns_steps.sql
+//
+
+var alterTableStepsAddColumnStepDependsOn = `
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT '';
+`
+
+var alterTableStepsAddColumnStepImage = `
+ALTER TABLE steps ADD COLUMN step_image TEXT NOT NULL DEFAULT '';
+`
+
+var alterTableStepsAddColumnStepDetached = `
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;
+`

store/shared/migrate/sqlite/files/016_add_columns_steps.sql

@@ -0,0 +1,11 @@
+-- name: alter-table-steps-add-column-step-depends-on
+
+ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT '';
+
+-- name: alter-table-steps-add-column-step-image
+
+ALTER TABLE steps ADD COLUMN step_image TEXT NOT NULL DEFAULT '';
+
+-- name: alter-table-steps-add-column-step-detached
+
+ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE;

store/stage/scan.go

@@ -112,6 +112,7 @@ func scanRow(scanner db.Scanner, dest *core.Stage) error {
 func scanRowStep(scanner db.Scanner, stage *core.Stage, step *nullStep) error {
 	depJSON := types.JSONText{}
 	labJSON := types.JSONText{}
+	stepDepJSON := types.JSONText{}
 	err := scanner.Scan(
 		&stage.ID,
 		&stage.RepoID,
@@ -151,9 +152,13 @@ func scanRowStep(scanner db.Scanner, stage *core.Stage, step *nullStep) error {
 		&step.Started,
 		&step.Stopped,
 		&step.Version,
+		&stepDepJSON,
+		&step.Image,
+		&step.Detached,
 	)
 	json.Unmarshal(depJSON, &stage.DependsOn)
 	json.Unmarshal(labJSON, &stage.Labels)
+	json.Unmarshal(stepDepJSON, &step.DependsOn)
 	return err
 }
 

store/stage/stage.go

@@ -323,6 +323,9 @@ SELECT
 ,step_started
 ,step_stopped
 ,step_version
+,step_depends_on
+,step_image
+,step_detached
 FROM stages
   LEFT JOIN steps
 	ON stages.stage_id=steps.step_stage_id
@@ -431,6 +434,9 @@ INSERT INTO steps (
 ,step_started
 ,step_stopped
 ,step_version
+,step_depends_on
+,step_image
+,step_detached
 ) VALUES (
  :step_stage_id
 ,:step_number
@@ -442,5 +448,8 @@ INSERT INTO steps (
 ,:step_started
 ,:step_stopped
 ,:step_version
+,:step_depends_on
+,:step_image
+,:step_detached
 )
 `

store/stage/type.go

@@ -16,8 +16,10 @@ package stage
 
 import (
 	"database/sql"
+	"encoding/json"
 
 	"github.com/drone/drone/core"
+	"github.com/jmoiron/sqlx/types"
 )
 
 type nullStep struct {
@@ -32,10 +34,16 @@ type nullStep struct {
 	Started   sql.NullInt64
 	Stopped   sql.NullInt64
 	Version   sql.NullInt64
+	DependsOn types.JSONText
+	Image     sql.NullString
+	Detached  sql.NullBool
 }
 
 func (s *nullStep) value() *core.Step {
-	return &core.Step{
+	var dependsOn []string
+	json.Unmarshal(s.DependsOn, &dependsOn)
+
+	step := &core.Step{
 		ID:        s.ID.Int64,
 		StageID:   s.StageID.Int64,
 		Number:    int(s.Number.Int64),
@@ -47,5 +55,10 @@ func (s *nullStep) value() *core.Step {
 		Started:   s.Started.Int64,
 		Stopped:   s.Stopped.Int64,
 		Version:   s.Version.Int64,
+		DependsOn: dependsOn,
+		Image:     s.Image.String,
+		Detached:  s.Detached.Bool,
 	}
+
+	return step
 }

store/step/scan.go

@@ -16,9 +16,11 @@ package step
 
 import (
 	"database/sql"
+	"encoding/json"
 
 	"github.com/drone/drone/core"
 	"github.com/drone/drone/store/shared/db"
+	"github.com/jmoiron/sqlx/types"
 )
 
 // helper function converts the Step structure to a set
@@ -36,13 +38,22 @@ func toParams(from *core.Step) map[string]interface{} {
 		"step_started":    from.Started,
 		"step_stopped":    from.Stopped,
 		"step_version":    from.Version,
+		"step_depends_on": encodeSlice(from.DependsOn),
+		"step_image":      from.Image,
+		"step_detached":   from.Detached,
 	}
 }
 
+func encodeSlice(v []string) types.JSONText {
+	raw, _ := json.Marshal(v)
+	return types.JSONText(raw)
+}
+
 // helper function scans the sql.Row and copies the column
 // values to the destination object.
 func scanRow(scanner db.Scanner, dest *core.Step) error {
-	return scanner.Scan(
+	depJSON := types.JSONText{}
+	err := scanner.Scan(
 		&dest.ID,
 		&dest.StageID,
 		&dest.Number,
@@ -54,7 +65,12 @@ func scanRow(scanner db.Scanner, dest *core.Step) error {
 		&dest.Started,
 		&dest.Stopped,
 		&dest.Version,
+		&depJSON,
+		&dest.Image,
+		&dest.Detached,
 	)
+	json.Unmarshal(depJSON, &dest.DependsOn)
+	return err
 }
 
 // helper function scans the sql.Row and copies the column

store/step/step.go

@@ -156,6 +156,9 @@ SELECT
 ,step_started
 ,step_stopped
 ,step_version
+,step_depends_on
+,step_image
+,step_detached
 `
 
 const queryKey = queryBase + `
@@ -185,6 +188,9 @@ SET
 ,step_started = :step_started
 ,step_stopped = :step_stopped
 ,step_version = :step_version_new
+,step_depends_on = :step_depends_on
+,step_image = :step_image
+,step_detached = :step_detached
 WHERE step_id = :step_id
   AND step_version = :step_version_old
 `
@@ -201,6 +207,9 @@ INSERT INTO steps (
 ,step_started
 ,step_stopped
 ,step_version
+,step_depends_on
+,step_image
+,step_detached
 ) VALUES (
  :step_stage_id
 ,:step_number
@@ -212,6 +221,9 @@ INSERT INTO steps (
 ,:step_started
 ,:step_stopped
 ,:step_version
+,:step_depends_on
+,:step_image
+,:step_detached
 )
 `
 

store/step/step_test.go

@@ -58,6 +58,9 @@ func testStepCreate(store *stepStore, stage *core.Stage) func(t *testing.T) {
 			ExitCode:  0,
 			Started:   1522878684,
 			Stopped:   0,
+			DependsOn: []string{"backend", "frontend"},
+			Image:     "ubuntu",
+			Detached:  false,
 		}
 		err := store.Create(noContext, item)
 		if err != nil {