diff --git a/CHANGELOG.md b/CHANGELOG.md index 1eb95661..be726954 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - endpoint to trigger new build for branch, by [@bradrydzewski](https://github.com/bradrydzewski). [#2679](https://github.com/drone/drone/issues/2679). - endpoint to trigger new build for branch and sha, by [@bradrydzewski](https://github.com/bradrydzewski). [#2679](https://github.com/drone/drone/issues/2679). - enable optional prometheus metrics guest access, by [@janberktold](https://github.com/janberktold) +- fallback to database when logs not found in s3, by [@bradrydzewski](https://github.com/bradrydzewski). [#2689](https://github.com/drone/drone/issues/2689). +- update drone-yaml to version 1.1.0 ### Fixed diff --git a/cmd/drone-server/inject_store.go b/cmd/drone-server/inject_store.go index 46906acb..34707627 100644 --- a/cmd/drone-server/inject_store.go +++ b/cmd/drone-server/inject_store.go @@ -84,12 +84,14 @@ func provideLogStore(db *db.DB, config config.Config) core.LogStore { if config.S3.Bucket == "" { return logs.New(db) } - return logs.NewS3Env( + s := logs.New(db) + p := logs.NewS3Env( config.S3.Bucket, config.S3.Prefix, config.S3.Endpoint, config.S3.PathStyle, ) + return logs.NewCombined(p, s) } // provideStageStore is a Wire provider function that provides a diff --git a/go.mod b/go.mod index d8ccaf41..9c56ffc5 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/drone/drone-go v1.0.5-0.20190427184118-618e4496482e github.com/drone/drone-runtime v1.0.6 github.com/drone/drone-ui v0.0.0-20190423061913-b758d7bee2eb - github.com/drone/drone-yaml v1.0.9 + github.com/drone/drone-yaml v1.1.0 github.com/drone/envsubst v1.0.1 github.com/drone/go-license v1.0.2 github.com/drone/go-login v1.0.4-0.20190311170324-2a4df4f242a2 @@ -88,7 +88,6 @@ require ( golang.org/x/text v0.3.0 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c google.golang.org/appengine v1.3.0 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/inf.v0 v0.9.1 gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20181130031204-d04500c8c3dd diff --git a/go.sum b/go.sum index 4dc73db6..7cb7dca9 100644 --- a/go.sum +++ b/go.sum @@ -15,8 +15,11 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= +github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/authcookie v0.0.0-20120917135355-fbdef6e99866 h1:98WJ4YCdjmB7uyrdT3P4A2Oa1hiRPKoa/0zInG6UnfQ= @@ -66,6 +69,8 @@ github.com/drone/drone-yaml v1.0.9-0.20190424150956-115b2ff5f99e h1:PvLIeYtS0/EX github.com/drone/drone-yaml v1.0.9-0.20190424150956-115b2ff5f99e/go.mod h1:1yrotgyD94qoYwgWWm71vAMbcw7Zd3gDersjeT9lYAk= github.com/drone/drone-yaml v1.0.9 h1:lVd8l1khAt7Ck+WMAjnRfEj+3jxLGOa8R7FITDh3jbw= github.com/drone/drone-yaml v1.0.9/go.mod h1:1yrotgyD94qoYwgWWm71vAMbcw7Zd3gDersjeT9lYAk= +github.com/drone/drone-yaml v1.1.0 h1:vY5AHfYtGTJD7mYtqtROiEpfjjjxnhbRfDxLfQ0zPzY= +github.com/drone/drone-yaml v1.1.0/go.mod h1:l/ehbHx9TGs4jgzhRnP5d+M9tmRsAmWyBHWAFEOXrk4= github.com/drone/envsubst v1.0.1 h1:NOOStingM2sbBwsIUeQkKUz8ShwCUzmqMxWrpXItfPE= github.com/drone/envsubst v1.0.1/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0= github.com/drone/go-license v1.0.2 h1:7OwndfYk+Lp/cGHkxe4HUn/Ysrrw3WYH2pnd99yrkok= @@ -152,6 +157,9 @@ github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= @@ -199,10 +207,14 @@ github.com/sirupsen/logrus v0.0.0-20181103062819-44067abb194b h1:dnSVC38LseSWVhj github.com/sirupsen/logrus v0.0.0-20181103062819-44067abb194b/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606 h1:dU9yXzNi9rl6Mou7+3npdfPyeFPb2+7BHs3zL47bhPY= github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= +github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d h1:3wDi6J5APMqaHBVPuVd7RmHD2gRTfqbdcVSpCNoUWtk= +github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490 h1:va0qYsIOza3Nlf2IncFyOql4/3XUq3vfge/Ad64bhlM= diff --git a/store/logs/combine.go b/store/logs/combine.go new file mode 100644 index 00000000..fe7a15ce --- /dev/null +++ b/store/logs/combine.go @@ -0,0 +1,61 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logs + +import ( + "context" + "io" + + "github.com/drone/drone/core" +) + +// NewCombined returns a new combined log store that will fallback +// to a secondary log store when necessary. This can be useful when +// migrating from database logs to s3, where logs for older builds +// are still being stored in the database, and newer logs in s3. +func NewCombined(primary, secondary core.LogStore) core.LogStore { + return &combined{ + primary: primary, + secondary: secondary, + } +} + +type combined struct { + primary, secondary core.LogStore +} + +func (s *combined) Find(ctx context.Context, step int64) (io.ReadCloser, error) { + rc, err := s.primary.Find(ctx, step) + if err == nil { + return rc, err + } + return s.secondary.Find(ctx, step) +} + +func (s *combined) Create(ctx context.Context, step int64, r io.Reader) error { + return s.primary.Create(ctx, step, r) +} + +func (s *combined) Update(ctx context.Context, step int64, r io.Reader) error { + return s.primary.Update(ctx, step, r) +} + +func (s *combined) Delete(ctx context.Context, step int64) error { + err := s.primary.Delete(ctx, step) + if err != nil { + err = s.secondary.Delete(ctx, step) + } + return err +}