From 8fe5a54b59d225eba5d127e42330e434589a3cb5 Mon Sep 17 00:00:00 2001 From: tphoney Date: Wed, 24 Feb 2021 15:02:42 +0000 Subject: [PATCH] (feat) adding depends_on, image and detached fields to step --- core/step.go | 25 +++++++----- scripts/build.sh | 0 store/shared/migrate/README.md | 32 +++++++++++++++ store/shared/migrate/mysql/ddl_gen.go | 28 +++++++++++++ .../mysql/files/016_add_columns_steps.sql | 11 +++++ store/shared/migrate/postgres/ddl_gen.go | 28 +++++++++++++ .../postgres/files/017_add_columns_steps.sql | 11 +++++ store/shared/migrate/sqlite/ddl_gen.go | 28 +++++++++++++ .../sqlite/files/016_add_columns_steps.sql | 11 +++++ store/stage/scan.go | 5 +++ store/stage/stage.go | 9 +++++ store/stage/type.go | 15 ++++++- store/step/scan.go | 40 +++++++++++++------ store/step/step.go | 12 ++++++ store/step/step_test.go | 17 ++++---- 15 files changed, 241 insertions(+), 31 deletions(-) mode change 100644 => 100755 scripts/build.sh create mode 100644 store/shared/migrate/README.md create mode 100644 store/shared/migrate/mysql/files/016_add_columns_steps.sql create mode 100644 store/shared/migrate/postgres/files/017_add_columns_steps.sql create mode 100644 store/shared/migrate/sqlite/files/016_add_columns_steps.sql diff --git a/core/step.go b/core/step.go index 76d505b4..49556432 100644 --- a/core/step.go +++ b/core/step.go @@ -19,17 +19,20 @@ import "context" type ( // Step represents an individual step in the stage. Step struct { - ID int64 `json:"id"` - StageID int64 `json:"step_id"` - Number int `json:"number"` - Name string `json:"name"` - Status string `json:"status"` - Error string `json:"error,omitempty"` - ErrIgnore bool `json:"errignore,omitempty"` - ExitCode int `json:"exit_code"` - Started int64 `json:"started,omitempty"` - Stopped int64 `json:"stopped,omitempty"` - Version int64 `json:"version"` + ID int64 `json:"id"` + StageID int64 `json:"step_id"` + Number int `json:"number"` + Name string `json:"name"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + ErrIgnore bool `json:"errignore,omitempty"` + ExitCode int `json:"exit_code"` + Started int64 `json:"started,omitempty"` + Stopped int64 `json:"stopped,omitempty"` + Version int64 `json:"version"` + DependsOn []string `json:"depends_on,omitempty"` + Image string `json:"image,omitempty"` + Detached bool `json:"detached,omitempty"` } // StepStore persists build step information to storage. diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 diff --git a/store/shared/migrate/README.md b/store/shared/migrate/README.md new file mode 100644 index 00000000..a31b7996 --- /dev/null +++ b/store/shared/migrate/README.md @@ -0,0 +1,32 @@ +# Building SQL DDL into Drone + +These folders contain the code for the different of databases that drone can use. They contain the SQL necessary to create the necessary tables and migrate between versions (IE the DDL). This SQL is generated into a go file and included as part of the Drone binary. + +## Making a changes to the database DDL + +Any new changes to the database structure are always put into a new SQL file. Follow the naming scheme in the `store/shared/migrate//files` of the SQL files by incrementing the number file name and give it a good description of what changes are being made. + +Changes will need to be implemented for all supported databases, making similar changes for eg Mysql/Postgres/Sqllite. + +**NB** Any changes to the database structure will need to be reflected for the relevant `struct` in the `core` directory. Changing the objects in the `store` directory for the ORM. Finally Possibly in the repositories github.com/drone/drone-go and github.com/drone/runner-go. + +## Generating Go from the SQL files + +To generate the go files you will need to install the golang command line tool `Togo` so it is on your users PATH. + +### Steps to install Togo + +``` bash +# in your workspace +git clone git@github.com:bradrydzewski/togo.git +cd togo +go get github.com/bradrydzewski/togo +``` + +### Generating go DDL + +Enter the desired database's implementation folder, and run the following. It will update the `ddl_gen.go` file. + +``` bash +go generate +``` diff --git a/store/shared/migrate/mysql/ddl_gen.go b/store/shared/migrate/mysql/ddl_gen.go index f60b6098..37af6360 100644 --- a/store/shared/migrate/mysql/ddl_gen.go +++ b/store/shared/migrate/mysql/ddl_gen.go @@ -164,6 +164,18 @@ var migrations = []struct { name: "create-index-template-namespace", stmt: createIndexTemplateNamespace, }, + { + name: "alter-table-steps-add-column-step-depends-on", + stmt: alterTableStepsAddColumnStepDependsOn, + }, + { + name: "alter-table-steps-add-column-step-image", + stmt: alterTableStepsAddColumnStepImage, + }, + { + name: "alter-table-steps-add-column-step-detached", + stmt: alterTableStepsAddColumnStepDetached, + }, } // Migrate performs the database migration. If the migration fails @@ -686,3 +698,19 @@ CREATE TABLE IF NOT EXISTS templates ( var createIndexTemplateNamespace = ` CREATE INDEX ix_template_namespace ON templates (template_namespace); ` + +// +// 016_add_columns_steps.sql +// + +var alterTableStepsAddColumnStepDependsOn = ` +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NULL; +` + +var alterTableStepsAddColumnStepImage = ` +ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT ''; +` + +var alterTableStepsAddColumnStepDetached = ` +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; +` diff --git a/store/shared/migrate/mysql/files/016_add_columns_steps.sql b/store/shared/migrate/mysql/files/016_add_columns_steps.sql new file mode 100644 index 00000000..6df6c67c --- /dev/null +++ b/store/shared/migrate/mysql/files/016_add_columns_steps.sql @@ -0,0 +1,11 @@ +-- name: alter-table-steps-add-column-step-depends-on + +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NULL; + +-- name: alter-table-steps-add-column-step-image + +ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT ''; + +-- name: alter-table-steps-add-column-step-detached + +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/store/shared/migrate/postgres/ddl_gen.go b/store/shared/migrate/postgres/ddl_gen.go index 337e1e44..149dac1a 100644 --- a/store/shared/migrate/postgres/ddl_gen.go +++ b/store/shared/migrate/postgres/ddl_gen.go @@ -156,6 +156,18 @@ var migrations = []struct { name: "create-table-template", stmt: createTableTemplate, }, + { + name: "alter-table-steps-add-column-step-depends-on", + stmt: alterTableStepsAddColumnStepDependsOn, + }, + { + name: "alter-table-steps-add-column-step-image", + stmt: alterTableStepsAddColumnStepImage, + }, + { + name: "alter-table-steps-add-column-step-detached", + stmt: alterTableStepsAddColumnStepDetached, + }, } // Migrate performs the database migration. If the migration fails @@ -658,3 +670,19 @@ CREATE TABLE IF NOT EXISTS templates ( CREATE INDEX IF NOT EXISTS ix_template_namespace ON templates (template_namespace); ` + +// +// 017_add_columns_steps.sql +// + +var alterTableStepsAddColumnStepDependsOn = ` +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT ''; +` + +var alterTableStepsAddColumnStepImage = ` +ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT ''; +` + +var alterTableStepsAddColumnStepDetached = ` +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; +` diff --git a/store/shared/migrate/postgres/files/017_add_columns_steps.sql b/store/shared/migrate/postgres/files/017_add_columns_steps.sql new file mode 100644 index 00000000..de8c0600 --- /dev/null +++ b/store/shared/migrate/postgres/files/017_add_columns_steps.sql @@ -0,0 +1,11 @@ +-- name: alter-table-steps-add-column-step-depends-on + +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT ''; + +-- name: alter-table-steps-add-column-step-image + +ALTER TABLE steps ADD COLUMN step_image VARCHAR(1000) NOT NULL DEFAULT ''; + +-- name: alter-table-steps-add-column-step-detached + +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/store/shared/migrate/sqlite/ddl_gen.go b/store/shared/migrate/sqlite/ddl_gen.go index 4afa29b9..e49312b6 100644 --- a/store/shared/migrate/sqlite/ddl_gen.go +++ b/store/shared/migrate/sqlite/ddl_gen.go @@ -156,6 +156,18 @@ var migrations = []struct { name: "create-table-templates", stmt: createTableTemplates, }, + { + name: "alter-table-steps-add-column-step-depends-on", + stmt: alterTableStepsAddColumnStepDependsOn, + }, + { + name: "alter-table-steps-add-column-step-image", + stmt: alterTableStepsAddColumnStepImage, + }, + { + name: "alter-table-steps-add-column-step-detached", + stmt: alterTableStepsAddColumnStepDetached, + }, } // Migrate performs the database migration. If the migration fails @@ -660,3 +672,19 @@ CREATE TABLE IF NOT EXISTS templates ( CREATE INDEX IF NOT EXISTS ix_template_namespace ON templates (template_namespace); ` + +// +// 016_add_columns_steps.sql +// + +var alterTableStepsAddColumnStepDependsOn = ` +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT ''; +` + +var alterTableStepsAddColumnStepImage = ` +ALTER TABLE steps ADD COLUMN step_image TEXT NOT NULL DEFAULT ''; +` + +var alterTableStepsAddColumnStepDetached = ` +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; +` diff --git a/store/shared/migrate/sqlite/files/016_add_columns_steps.sql b/store/shared/migrate/sqlite/files/016_add_columns_steps.sql new file mode 100644 index 00000000..7e01996d --- /dev/null +++ b/store/shared/migrate/sqlite/files/016_add_columns_steps.sql @@ -0,0 +1,11 @@ +-- name: alter-table-steps-add-column-step-depends-on + +ALTER TABLE steps ADD COLUMN step_depends_on TEXT NOT NULL DEFAULT ''; + +-- name: alter-table-steps-add-column-step-image + +ALTER TABLE steps ADD COLUMN step_image TEXT NOT NULL DEFAULT ''; + +-- name: alter-table-steps-add-column-step-detached + +ALTER TABLE steps ADD COLUMN step_detached BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/store/stage/scan.go b/store/stage/scan.go index a488bc2b..ebe54746 100644 --- a/store/stage/scan.go +++ b/store/stage/scan.go @@ -112,6 +112,7 @@ func scanRow(scanner db.Scanner, dest *core.Stage) error { func scanRowStep(scanner db.Scanner, stage *core.Stage, step *nullStep) error { depJSON := types.JSONText{} labJSON := types.JSONText{} + stepDepJSON := types.JSONText{} err := scanner.Scan( &stage.ID, &stage.RepoID, @@ -151,9 +152,13 @@ func scanRowStep(scanner db.Scanner, stage *core.Stage, step *nullStep) error { &step.Started, &step.Stopped, &step.Version, + &stepDepJSON, + &step.Image, + &step.Detached, ) json.Unmarshal(depJSON, &stage.DependsOn) json.Unmarshal(labJSON, &stage.Labels) + json.Unmarshal(stepDepJSON, &step.DependsOn) return err } diff --git a/store/stage/stage.go b/store/stage/stage.go index fbf496d8..7791d6a0 100644 --- a/store/stage/stage.go +++ b/store/stage/stage.go @@ -323,6 +323,9 @@ SELECT ,step_started ,step_stopped ,step_version +,step_depends_on +,step_image +,step_detached FROM stages LEFT JOIN steps ON stages.stage_id=steps.step_stage_id @@ -431,6 +434,9 @@ INSERT INTO steps ( ,step_started ,step_stopped ,step_version +,step_depends_on +,step_image +,step_detached ) VALUES ( :step_stage_id ,:step_number @@ -442,5 +448,8 @@ INSERT INTO steps ( ,:step_started ,:step_stopped ,:step_version +,:step_depends_on +,:step_image +,:step_detached ) ` diff --git a/store/stage/type.go b/store/stage/type.go index baa00d39..93c36acc 100644 --- a/store/stage/type.go +++ b/store/stage/type.go @@ -16,8 +16,10 @@ package stage import ( "database/sql" + "encoding/json" "github.com/drone/drone/core" + "github.com/jmoiron/sqlx/types" ) type nullStep struct { @@ -32,10 +34,16 @@ type nullStep struct { Started sql.NullInt64 Stopped sql.NullInt64 Version sql.NullInt64 + DependsOn types.JSONText + Image sql.NullString + Detached sql.NullBool } func (s *nullStep) value() *core.Step { - return &core.Step{ + var dependsOn []string + json.Unmarshal(s.DependsOn, &dependsOn) + + step := &core.Step{ ID: s.ID.Int64, StageID: s.StageID.Int64, Number: int(s.Number.Int64), @@ -47,5 +55,10 @@ func (s *nullStep) value() *core.Step { Started: s.Started.Int64, Stopped: s.Stopped.Int64, Version: s.Version.Int64, + DependsOn: dependsOn, + Image: s.Image.String, + Detached: s.Detached.Bool, } + + return step } diff --git a/store/step/scan.go b/store/step/scan.go index d5e014f5..d6dd9470 100644 --- a/store/step/scan.go +++ b/store/step/scan.go @@ -16,33 +16,44 @@ package step import ( "database/sql" + "encoding/json" "github.com/drone/drone/core" "github.com/drone/drone/store/shared/db" + "github.com/jmoiron/sqlx/types" ) // helper function converts the Step structure to a set // of named query parameters. func toParams(from *core.Step) map[string]interface{} { return map[string]interface{}{ - "step_id": from.ID, - "step_stage_id": from.StageID, - "step_number": from.Number, - "step_name": from.Name, - "step_status": from.Status, - "step_error": from.Error, - "step_errignore": from.ErrIgnore, - "step_exit_code": from.ExitCode, - "step_started": from.Started, - "step_stopped": from.Stopped, - "step_version": from.Version, + "step_id": from.ID, + "step_stage_id": from.StageID, + "step_number": from.Number, + "step_name": from.Name, + "step_status": from.Status, + "step_error": from.Error, + "step_errignore": from.ErrIgnore, + "step_exit_code": from.ExitCode, + "step_started": from.Started, + "step_stopped": from.Stopped, + "step_version": from.Version, + "step_depends_on": encodeSlice(from.DependsOn), + "step_image": from.Image, + "step_detached": from.Detached, } } +func encodeSlice(v []string) types.JSONText { + raw, _ := json.Marshal(v) + return types.JSONText(raw) +} + // helper function scans the sql.Row and copies the column // values to the destination object. func scanRow(scanner db.Scanner, dest *core.Step) error { - return scanner.Scan( + depJSON := types.JSONText{} + err := scanner.Scan( &dest.ID, &dest.StageID, &dest.Number, @@ -54,7 +65,12 @@ func scanRow(scanner db.Scanner, dest *core.Step) error { &dest.Started, &dest.Stopped, &dest.Version, + &depJSON, + &dest.Image, + &dest.Detached, ) + json.Unmarshal(depJSON, &dest.DependsOn) + return err } // helper function scans the sql.Row and copies the column diff --git a/store/step/step.go b/store/step/step.go index e43350bd..5ade4da1 100644 --- a/store/step/step.go +++ b/store/step/step.go @@ -156,6 +156,9 @@ SELECT ,step_started ,step_stopped ,step_version +,step_depends_on +,step_image +,step_detached ` const queryKey = queryBase + ` @@ -185,6 +188,9 @@ SET ,step_started = :step_started ,step_stopped = :step_stopped ,step_version = :step_version_new +,step_depends_on = :step_depends_on +,step_image = :step_image +,step_detached = :step_detached WHERE step_id = :step_id AND step_version = :step_version_old ` @@ -201,6 +207,9 @@ INSERT INTO steps ( ,step_started ,step_stopped ,step_version +,step_depends_on +,step_image +,step_detached ) VALUES ( :step_stage_id ,:step_number @@ -212,6 +221,9 @@ INSERT INTO steps ( ,:step_started ,:step_stopped ,:step_version +,:step_depends_on +,:step_image +,:step_detached ) ` diff --git a/store/step/step_test.go b/store/step/step_test.go index 92906cfd..7f2b7a7d 100644 --- a/store/step/step_test.go +++ b/store/step/step_test.go @@ -51,13 +51,16 @@ func TestStep(t *testing.T) { func testStepCreate(store *stepStore, stage *core.Stage) func(t *testing.T) { return func(t *testing.T) { item := &core.Step{ - StageID: stage.ID, - Number: 2, - Name: "clone", - Status: core.StatusRunning, - ExitCode: 0, - Started: 1522878684, - Stopped: 0, + StageID: stage.ID, + Number: 2, + Name: "clone", + Status: core.StatusRunning, + ExitCode: 0, + Started: 1522878684, + Stopped: 0, + DependsOn: []string{"backend", "frontend"}, + Image: "ubuntu", + Detached: false, } err := store.Create(noContext, item) if err != nil {