fixes #2689

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - endpoint to trigger new build for branch, by [@bradrydzewski](https://github.com/bradrydzewski). [#2679](https://github.com/drone/drone/issues/2679).
 - endpoint to trigger new build for branch and sha, by [@bradrydzewski](https://github.com/bradrydzewski). [#2679](https://github.com/drone/drone/issues/2679).
 - enable optional prometheus metrics guest access, by [@janberktold](https://github.com/janberktold)
+- fallback to database when logs not found in s3, by [@bradrydzewski](https://github.com/bradrydzewski). [#2689](https://github.com/drone/drone/issues/2689).
+- update drone-yaml to version 1.1.0
 
 ### Fixed
 

cmd/drone-server/inject_store.go

@@ -84,12 +84,14 @@ func provideLogStore(db *db.DB, config config.Config) core.LogStore {
 	if config.S3.Bucket == "" {
 		return logs.New(db)
 	}
-	return logs.NewS3Env(
+	s := logs.New(db)
+	p := logs.NewS3Env(
 		config.S3.Bucket,
 		config.S3.Prefix,
 		config.S3.Endpoint,
 		config.S3.PathStyle,
 	)
+	return logs.NewCombined(p, s)
 }
 
 // provideStageStore is a Wire provider function that provides a

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/drone/drone-go v1.0.5-0.20190427184118-618e4496482e
 	github.com/drone/drone-runtime v1.0.6
 	github.com/drone/drone-ui v0.0.0-20190423061913-b758d7bee2eb
-	github.com/drone/drone-yaml v1.0.9
+	github.com/drone/drone-yaml v1.1.0
 	github.com/drone/envsubst v1.0.1
 	github.com/drone/go-license v1.0.2
 	github.com/drone/go-login v1.0.4-0.20190311170324-2a4df4f242a2
@@ -88,7 +88,6 @@ require (
 	golang.org/x/text v0.3.0
 	golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
 	google.golang.org/appengine v1.3.0
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20181130031204-d04500c8c3dd

go.sum

@@ -15,8 +15,11 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ=
 github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w=
+github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8=
+github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI=
 github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dchest/authcookie v0.0.0-20120917135355-fbdef6e99866 h1:98WJ4YCdjmB7uyrdT3P4A2Oa1hiRPKoa/0zInG6UnfQ=
@@ -66,6 +69,8 @@ github.com/drone/drone-yaml v1.0.9-0.20190424150956-115b2ff5f99e h1:PvLIeYtS0/EX
 github.com/drone/drone-yaml v1.0.9-0.20190424150956-115b2ff5f99e/go.mod h1:1yrotgyD94qoYwgWWm71vAMbcw7Zd3gDersjeT9lYAk=
 github.com/drone/drone-yaml v1.0.9 h1:lVd8l1khAt7Ck+WMAjnRfEj+3jxLGOa8R7FITDh3jbw=
 github.com/drone/drone-yaml v1.0.9/go.mod h1:1yrotgyD94qoYwgWWm71vAMbcw7Zd3gDersjeT9lYAk=
+github.com/drone/drone-yaml v1.1.0 h1:vY5AHfYtGTJD7mYtqtROiEpfjjjxnhbRfDxLfQ0zPzY=
+github.com/drone/drone-yaml v1.1.0/go.mod h1:l/ehbHx9TGs4jgzhRnP5d+M9tmRsAmWyBHWAFEOXrk4=
 github.com/drone/envsubst v1.0.1 h1:NOOStingM2sbBwsIUeQkKUz8ShwCUzmqMxWrpXItfPE=
 github.com/drone/envsubst v1.0.1/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0=
 github.com/drone/go-license v1.0.2 h1:7OwndfYk+Lp/cGHkxe4HUn/Ysrrw3WYH2pnd99yrkok=
@@ -152,6 +157,9 @@ github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM=
 github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
@@ -199,10 +207,14 @@ github.com/sirupsen/logrus v0.0.0-20181103062819-44067abb194b h1:dnSVC38LseSWVhj
 github.com/sirupsen/logrus v0.0.0-20181103062819-44067abb194b/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606 h1:dU9yXzNi9rl6Mou7+3npdfPyeFPb2+7BHs3zL47bhPY=
 github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=
+github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d h1:3wDi6J5APMqaHBVPuVd7RmHD2gRTfqbdcVSpCNoUWtk=
+github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo=
 golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181012144002-a92615f3c490 h1:va0qYsIOza3Nlf2IncFyOql4/3XUq3vfge/Ad64bhlM=

store/logs/combine.go

@@ -0,0 +1,61 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package logs
+
+import (
+	"context"
+	"io"
+
+	"github.com/drone/drone/core"
+)
+
+// NewCombined returns a new combined log store that will fallback
+// to a secondary log store when necessary. This can be useful when
+// migrating from database logs to s3, where logs for older builds
+// are still being stored in the database, and newer logs in s3.
+func NewCombined(primary, secondary core.LogStore) core.LogStore {
+	return &combined{
+		primary:   primary,
+		secondary: secondary,
+	}
+}
+
+type combined struct {
+	primary, secondary core.LogStore
+}
+
+func (s *combined) Find(ctx context.Context, step int64) (io.ReadCloser, error) {
+	rc, err := s.primary.Find(ctx, step)
+	if err == nil {
+		return rc, err
+	}
+	return s.secondary.Find(ctx, step)
+}
+
+func (s *combined) Create(ctx context.Context, step int64, r io.Reader) error {
+	return s.primary.Create(ctx, step, r)
+}
+
+func (s *combined) Update(ctx context.Context, step int64, r io.Reader) error {
+	return s.primary.Update(ctx, step, r)
+}
+
+func (s *combined) Delete(ctx context.Context, step int64) error {
+	err := s.primary.Delete(ctx, step)
+	if err != nil {
+		err = s.secondary.Delete(ctx, step)
+	}
+	return err
+}