From 40d09839f7bf198f053afd135e7fa7577a692917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Thu, 22 Jul 2021 15:03:57 +0200 Subject: [PATCH 1/8] pubsub: redis implementation --- cmd/drone-server/config/config.go | 6 ++ cmd/drone-server/inject_store.go | 17 ++++ cmd/drone-server/wire_gen.go | 9 ++- pubsub/hub.go | 4 +- pubsub/pubsub.go | 31 ++++++++ pubsub/redis.go | 126 ++++++++++++++++++++++++++++++ 6 files changed, 190 insertions(+), 3 deletions(-) create mode 100644 pubsub/pubsub.go create mode 100644 pubsub/redis.go diff --git a/cmd/drone-server/config/config.go b/cmd/drone-server/config/config.go index fb14fe1c..29b7eae0 100644 --- a/cmd/drone-server/config/config.go +++ b/cmd/drone-server/config/config.go @@ -63,6 +63,7 @@ type ( Logging Logging Prometheus Prometheus Proxy Proxy + Redis Redis Registration Registration Registries Registries Repository Repository @@ -166,6 +167,11 @@ type ( EnableAnonymousAccess bool `envconfig:"DRONE_PROMETHEUS_ANONYMOUS_ACCESS" default:"false"` } + // Redis provides the redis configuration. + Redis struct { + ConnectionString string `envconfig:"DRONE_REDIS_CONNECTION"` + } + // Repository provides the repository configuration. Repository struct { Filter []string `envconfig:"DRONE_REPOSITORY_FILTER"` diff --git a/cmd/drone-server/inject_store.go b/cmd/drone-server/inject_store.go index 0e06e9b0..d9116856 100644 --- a/cmd/drone-server/inject_store.go +++ b/cmd/drone-server/inject_store.go @@ -34,6 +34,7 @@ import ( "github.com/drone/drone/store/template" "github.com/drone/drone/store/user" + "github.com/go-redis/redis/v8" "github.com/google/wire" "github.com/sirupsen/logrus" ) @@ -55,6 +56,7 @@ var storeSet = wire.NewSet( global.New, step.New, template.New, + provideRedisClient, ) // provideDatabase is a Wire provider function that provides a @@ -177,3 +179,18 @@ func provideUserStore(db *db.DB, enc encrypt.Encrypter, config config.Config) co metric.UserCount(users) return users } + +func provideRedisClient(config config.Config) (rdb *redis.Client, err error) { + if config.Redis.ConnectionString == "" { + return + } + + options, err := redis.ParseURL(config.Redis.ConnectionString) + if err != nil { + return + } + + rdb = redis.NewClient(options) + + return +} diff --git a/cmd/drone-server/wire_gen.go b/cmd/drone-server/wire_gen.go index 94eaefc9..d65d65b7 100644 --- a/cmd/drone-server/wire_gen.go +++ b/cmd/drone-server/wire_gen.go @@ -55,7 +55,14 @@ func InitializeApplication(config2 config.Config) (application, error) { cronStore := cron.New(db) repositoryStore := provideRepoStore(db) buildStore := provideBuildStore(db) - corePubsub := pubsub.New() + redisClient, err := provideRedisClient(config2) + if err != nil { + return application{}, err + } + corePubsub, err := pubsub.New(redisClient) + if err != nil { + return application{}, err + } stageStore := provideStageStore(db) scheduler := provideScheduler(stageStore, config2) statusService := provideStatusService(client, renewer, config2) diff --git a/pubsub/hub.go b/pubsub/hub.go index 6249a771..3445e7bc 100644 --- a/pubsub/hub.go +++ b/pubsub/hub.go @@ -27,8 +27,8 @@ type hub struct { subs map[*subscriber]struct{} } -// New creates a new publish subscriber. -func New() core.Pubsub { +// newHub creates a new publish subscriber. +func newHub() core.Pubsub { return &hub{ subs: map[*subscriber]struct{}{}, } diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go new file mode 100644 index 00000000..201f42e0 --- /dev/null +++ b/pubsub/pubsub.go @@ -0,0 +1,31 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pubsub + +import ( + "github.com/drone/drone/core" + + "github.com/go-redis/redis/v8" +) + +// New creates a new publish subscriber. If Redis client passed as parameter is not nil it uses +// a Redis implementation, otherwise it uses an in-memory implementation. +func New(rdb *redis.Client) (core.Pubsub, error) { + if rdb != nil { + return newRedis(rdb) + } + + return newHub(), nil +} diff --git a/pubsub/redis.go b/pubsub/redis.go new file mode 100644 index 00000000..c322b118 --- /dev/null +++ b/pubsub/redis.go @@ -0,0 +1,126 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pubsub + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/drone/drone/core" + + "github.com/go-redis/redis/v8" +) + +const channelPubSub = "drone-events" + +type hubRedis struct { + rdb *redis.Client +} + +func (h *hubRedis) Publish(ctx context.Context, e *core.Message) (err error) { + data, err := json.Marshal(e) + if err != nil { + return + } + + _, err = h.rdb.Publish(ctx, channelPubSub, data).Result() + if err != nil { + return + } + + return +} + +func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan error) { + messageCh := make(chan *core.Message, 100) + errCh := make(chan error) + + go func() { + pubsub := h.rdb.Subscribe(ctx, channelPubSub) + ch := pubsub.Channel(redis.WithChannelSize(100)) + + defer func() { + _ = pubsub.Close() + close(messageCh) + close(errCh) + }() + + err := pubsub.Ping(ctx) + if err != nil { + errCh <- err + return + } + + for { + select { + case m, ok := <-ch: + if ok { + message := &core.Message{} + err = json.Unmarshal([]byte(m.Payload), message) + if err != nil { + fmt.Printf("error@pubsub: failed to unmarshal a message. %s\n", err) + continue + } + messageCh <- message + } else { + errCh <- fmt.Errorf("redis pubsub channel=%s closed", channelPubSub) + return + } + case <-ctx.Done(): + return + } + } + }() + + return messageCh, errCh +} + +func (h *hubRedis) Subscribers() int { + ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFunc() + + v, err := h.rdb.Do(ctx, "pubsub", "numsub", channelPubSub).Result() + if err != nil { + fmt.Printf("error@pubsub: failed to get number of subscribers. %s\n", err) + return -1 + } + + values, ok := v.([]interface{}) // the result should be: [, ] + if !ok || len(values) != 2 { + return 0 + } + + if subscriberCount, ok := values[1].(int64); ok { + return int(subscriberCount) + } + + return 0 +} + +func newRedis(rdb *redis.Client) (ps core.Pubsub, err error) { + _, err = rdb.Ping(context.Background()).Result() + if err != nil { + err = fmt.Errorf("redis not accessibe: %w", err) + return + } + + ps = &hubRedis{ + rdb: rdb, + } + + return +} From ebbe87e4c64e0094f31fcd2800607455adb4a20a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Thu, 22 Jul 2021 15:06:48 +0200 Subject: [PATCH 2/8] added redis client library to go.mod --- go.mod | 29 ++--- go.sum | 328 ++++++++++++--------------------------------------------- 2 files changed, 73 insertions(+), 284 deletions(-) diff --git a/go.mod b/go.mod index 508e9a8c..58d0ca9a 100644 --- a/go.mod +++ b/go.mod @@ -25,51 +25,36 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/go-chi/chi v3.3.3+incompatible github.com/go-chi/cors v1.0.0 - github.com/go-ini/ini v1.39.0 // indirect + github.com/go-redis/redis/v8 v8.11.0 github.com/go-sql-driver/mysql v1.4.0 github.com/golang/mock v1.3.1 - github.com/google/go-cmp v0.4.0 + github.com/google/go-cmp v0.5.6 github.com/google/go-jsonnet v0.17.0 github.com/google/wire v0.2.1 - github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 // indirect github.com/gosimple/slug v1.3.0 github.com/h2non/gock v1.0.15 - github.com/hashicorp/consul/api v1.5.0 // indirect - github.com/hashicorp/go-hclog v0.14.1 // indirect - github.com/hashicorp/go-immutable-radix v1.2.0 // indirect - github.com/hashicorp/go-msgpack v1.1.5 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-multierror v1.0.0 - github.com/hashicorp/go-plugin v1.3.0 // indirect github.com/hashicorp/go-retryablehttp v0.5.4 - github.com/hashicorp/go-version v1.2.1 // indirect github.com/hashicorp/golang-lru v0.5.1 - github.com/hashicorp/nomad v0.0.0-20190125003214-134391155854 - github.com/hashicorp/raft v1.1.2 // indirect - github.com/hashicorp/vault/api v1.0.4 // indirect github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 github.com/joho/godotenv v1.3.0 github.com/kelseyhightower/envconfig v1.3.0 + github.com/kr/pretty v0.2.0 // indirect github.com/lib/pq v1.1.0 github.com/mattn/go-sqlite3 v1.9.0 - github.com/mitchellh/go-testing-interface v1.14.1 // indirect - github.com/mitchellh/hashstructure v1.0.0 // indirect github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 github.com/prometheus/client_golang v0.9.2 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be // indirect github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 github.com/segmentio/ksuid v1.0.2 github.com/sirupsen/logrus v1.6.0 - github.com/smartystreets/goconvey v1.6.4 // indirect - github.com/ugorji/go v1.1.7 // indirect github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606 go.starlark.net v0.0.0-20201118183435-e55f603d8c79 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 - golang.org/x/sync v0.0.0-20190423024810-112230192c58 - gopkg.in/ini.v1 v1.57.0 // indirect - gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.0.0-20181130031204-d04500c8c3dd - k8s.io/apimachinery v0.0.0-20181204150028-eb8c8024849b - k8s.io/client-go v10.0.0+incompatible + golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect + gopkg.in/yaml.v2 v2.3.0 ) replace github.com/h2non/gock => gopkg.in/h2non/gock.v1 v1.0.14 diff --git a/go.sum b/go.sum index 59b82bea..8d531c4c 100644 --- a/go.sum +++ b/go.sum @@ -25,36 +25,26 @@ github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6L github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Microsoft/go-winio v0.4.11 h1:zoIOcVf0xPN1tnMVbTtEdI+P8OofVk3NObnwOQ6nK2Q= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= -github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f h1:y2hSFdXeA1y5z5f0vfNO0Dg5qVY036qzlz3Pds0B92o= github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.15.57 h1:inht07/mRNnvV4uAjjVgTVD7/rF+j0mXllYcNQxDgGA= -github.com/aws/aws-sdk-go v1.15.57/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.37.3 h1:1f0groABc4AuapskpHf6EBRaG2tqw0Sx3ebCMwfp1Ys= github.com/aws/aws-sdk-go v1.37.3/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY= @@ -72,6 +62,8 @@ github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9 h1:74lLNRzvsdIlkTgfD github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -81,53 +73,11 @@ github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/drone/drone-go v1.3.2-0.20200826185551-24929e4d2cfc h1:6AXXCMPilpJqwNqUfG6Zgwakr7HCwnialFiQ/AbqYyQ= -github.com/drone/drone-go v1.3.2-0.20200826185551-24929e4d2cfc/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg= github.com/drone/drone-go v1.4.1-0.20201109202657-b9e58bbbcf27 h1:58xKlW/Kwp/Apz+R5qNGzBUIzfq1Z57L7Udz1B6bgWE= github.com/drone/drone-go v1.4.1-0.20201109202657-b9e58bbbcf27/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg= github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs= github.com/drone/drone-runtime v1.1.1-0.20200623162453-61e33e2cab5d h1:P5HI/Y9hARTZ3F3EKs0kYijhjXZWQRQHYn1neTi0pWM= github.com/drone/drone-runtime v1.1.1-0.20200623162453-61e33e2cab5d/go.mod h1:4/2QToW5+HGD0y1sTw7X35W1f7YINS14UfDY4isggT8= -github.com/drone/drone-ui v0.0.0-20200701170131-2b91a041998b h1:8VfphhR5arTUOFGf8KpNkEBf2z99kuC/YdnasY75Xus= -github.com/drone/drone-ui v0.0.0-20200701170131-2b91a041998b/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20201110214517-ac1349fcc19c h1:RHVLOVo6vC/p3i64XMQ6lcURQCeqUbg+7vyhCAy8hq0= -github.com/drone/drone-ui v0.0.0-20201110214517-ac1349fcc19c/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318043635-d92124ef8c6b h1:SR3ORKKtSGLA2oGHBr6tP+KaHRB7JSveYreZVxnfQgA= -github.com/drone/drone-ui v0.0.0-20210318043635-d92124ef8c6b/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318051923-d744fdf178d7 h1:KpTwlg8z0xcT7UZXTzK8G0r5+1CeXKkglZwsg6wHuK4= -github.com/drone/drone-ui v0.0.0-20210318051923-d744fdf178d7/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318173335-90cdf440f023 h1:xy7suHSQ4wEuOa8EnCmBpuMEXcfG66QJxY0rdDG9w3c= -github.com/drone/drone-ui v0.0.0-20210318173335-90cdf440f023/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318184040-660cf374c83a h1:b36lw0bXDWRB8miPIA59SUgpV+mTvoUX7otADm8SwyQ= -github.com/drone/drone-ui v0.0.0-20210318184040-660cf374c83a/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318190358-0fd3dd080c5b h1:CfVtYTmPVUm5x2UTUvFF8NtQMYWwlVWz9OiRfxuiLQg= -github.com/drone/drone-ui v0.0.0-20210318190358-0fd3dd080c5b/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210318211453-780e11c45d95 h1:8JsUeiL9slNUbMUYGINI0gi2owD/0JnzKgNrv6EmxHY= -github.com/drone/drone-ui v0.0.0-20210318211453-780e11c45d95/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210319172440-b892bff542b5 h1:6mjZNWxqVKkBt1Pc2ChdMFOzwa8+7oTVX4dTLnEx4dA= -github.com/drone/drone-ui v0.0.0-20210319172440-b892bff542b5/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210322195809-43fc5b050223 h1:lyWHV4jHKzF8kKBrEXjlZDTyuuXuAQ2BabeURGt5hLo= -github.com/drone/drone-ui v0.0.0-20210322195809-43fc5b050223/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210324203842-aaae3627e294 h1:ifbOWfDF/4zNxXW69f4XL06D7aA7cdp5U0moNNh60nw= -github.com/drone/drone-ui v0.0.0-20210324203842-aaae3627e294/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210331233327-772f24b7f519 h1:zEPxQwLA/dI6sQ9htLPWcDT+EKms09w4TSAYk1lOQiA= -github.com/drone/drone-ui v0.0.0-20210331233327-772f24b7f519/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210427231613-96a9f6e17fb7 h1:EUvu5J51N2diuFqLjFTiu65m0EK0TlLD66qDyi43tVE= -github.com/drone/drone-ui v0.0.0-20210427231613-96a9f6e17fb7/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210505020539-1b93fc25c7d8 h1:mIFBOdP8Tif/4li4bmiVjqp6vbxMpTHOuZR80eypm6A= -github.com/drone/drone-ui v0.0.0-20210505020539-1b93fc25c7d8/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210512200715-d96f1e26d448 h1:skfTTwMRWSSi3Dv5NrpU8mJn7faccG5q+lqiBQikLiw= -github.com/drone/drone-ui v0.0.0-20210512200715-d96f1e26d448/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210602131102-d9e6fc7e8e68 h1:MFiB2sySRfQfdGyzCw6ii45IVgNXS1Ho+YmKxdmp9eU= -github.com/drone/drone-ui v0.0.0-20210602131102-d9e6fc7e8e68/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210616003421-c6bb64fb3c3b h1:w7zIfuX1sv491AC0hGQzbtGQ/IXfiQmLbBKVrIuEC+0= -github.com/drone/drone-ui v0.0.0-20210616003421-c6bb64fb3c3b/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210616023721-d11e2b9d4c32 h1:pnrIQX6PDZCXs8uh6kvWrjXwhw3ZSg6D1Yp6W+Pe5Pw= -github.com/drone/drone-ui v0.0.0-20210616023721-d11e2b9d4c32/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210623033012-76a6768a4050 h1:VjS2w8DtcdOajM11UYc2bzlqXONna5MnyToMWO6uCJk= -github.com/drone/drone-ui v0.0.0-20210623033012-76a6768a4050/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= -github.com/drone/drone-ui v0.0.0-20210623224836-9a5c77ebfdb7 h1:gqEVIHEIdzBtoeL0fE33Ba+OD1QARTGRZTw4jrGOxCk= -github.com/drone/drone-ui v0.0.0-20210623224836-9a5c77ebfdb7/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= github.com/drone/drone-ui v0.0.0-20210702192143-f81f74399385 h1:nhQAc+Zo1cwgHXMS3O0hUcPl0k1wqGv/Ig+h0+8nLps= github.com/drone/drone-ui v0.0.0-20210702192143-f81f74399385/go.mod h1:NBtVWW7NNJpD9+huMD/5TAE1db2nrEh0i35/9Rf1MPI= github.com/drone/drone-yaml v1.2.4-0.20200326192514-6f4d6dfb39e4 h1:XsstoCeXC2t8lA9OLTdoFwckaptqahxwjCWsenySfX8= @@ -138,20 +88,6 @@ github.com/drone/go-license v1.0.2 h1:7OwndfYk+Lp/cGHkxe4HUn/Ysrrw3WYH2pnd99yrko github.com/drone/go-license v1.0.2/go.mod h1:fGRHf+F1cEaw3YVYiJ6js3G3dVhcxyS617RnNRUMsms= github.com/drone/go-login v1.0.4-0.20190311170324-2a4df4f242a2 h1:RGpgNkowJc5LAVn/ZONx70qmnaTA0z/3hHPzTBdAEO8= github.com/drone/go-login v1.0.4-0.20190311170324-2a4df4f242a2/go.mod h1:FLxy9vRzLbyBxoCJYxGbG9R0WGn6OyuvBmAtYNt43uw= -github.com/drone/go-scm v1.7.1-0.20200621203823-3731ec1f1136 h1:RKSqrIGBdttX999GgY7J2n1zIJ3QVXH8coI4ObTBB9E= -github.com/drone/go-scm v1.7.1-0.20200621203823-3731ec1f1136/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.7.2-0.20201015134822-a014bb19b1df h1:CzwzLwv6h9HMJSph4hC9pJzcaPhExUUOf7s3KEu9lRw= -github.com/drone/go-scm v1.7.2-0.20201015134822-a014bb19b1df/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.7.2-0.20201028160627-427b8a85897c h1:3Dv6guONE4nry6fvwEvFz/+pnC0tsMudkrz6CwRO3KM= -github.com/drone/go-scm v1.7.2-0.20201028160627-427b8a85897c/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.7.2-0.20201111225713-c0438b46084b h1:ivLeFPmHN+9sLMVAF7HvgvEglU5tzoqlzePLY5zKPo8= -github.com/drone/go-scm v1.7.2-0.20201111225713-c0438b46084b/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.8.0 h1:kDHu38a11loKf6uaBu75TmY1YPwsSaZdseET738Oy0o= -github.com/drone/go-scm v1.8.0/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.9.0 h1:KgaGREXA7Ncu4ccdnk7p93hJwE8B8GLaBHfRprwtUCE= -github.com/drone/go-scm v1.9.0/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= -github.com/drone/go-scm v1.15.0 h1:yBO6lcCeegbEuEaH0QUvJmBVQS/RpYKzuzULHHMT2A4= -github.com/drone/go-scm v1.15.0/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= github.com/drone/go-scm v1.15.1 h1:35m/CcHkYjQ4BlOM7rIIwrki6uDUbUH+Kkb9rv6om3M= github.com/drone/go-scm v1.15.1/go.mod h1:lXwfbyrIJwFFME5TpzavkwO2T5X8yBK6t6cve7g91x0= github.com/drone/signal v1.0.0 h1:NrnM2M/4yAuU/tXs6RP1a1ZfxnaHwYkd0kJurA1p6uI= @@ -159,30 +95,25 @@ github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGn github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8= github.com/go-chi/chi v3.3.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/cors v1.0.0 h1:e6x8k7uWbUwYs+aXDoiUzeQFT6l0cygBYyNhD7/1Tg0= github.com/go-chi/cors v1.0.0/go.mod h1:K2Yje0VW/SJzxiyMYu6iPQYa7hMjQX2i/F491VChg1I= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ini/ini v1.39.0 h1:/CyW/jTlZLjuzy52jc1XnhJm6IUKEuunpJFpecywNeI= -github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= +github.com/go-redis/redis/v8 v8.11.0 h1:O1Td0mQ8UFChQ3N9zFQqo6kTU2cJ+/it88gDB+zg0wo= +github.com/go-redis/redis/v8 v8.11.0/go.mod h1:DLomh7y2e3ggQXQLd1YgmvIfecPJoFl7WU5SOQ/r06M= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506 h1:zDlw+wgyXdfkRuvFCdEDUiPLmZp2cvf/dWHazY0a5VM= github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= @@ -191,121 +122,56 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-jsonnet v0.16.0 h1:Nb4EEOp+rdeGGyB1rQ5eisgSAqrTnhf9ip+X6lzZbY0= -github.com/google/go-jsonnet v0.16.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-jsonnet v0.17.0 h1:/9NIEfhK1NQRKl3sP2536b2+x5HnZMdql7x3yK/l8JY= github.com/google/go-jsonnet v0.17.0/go.mod h1:sOcuej3UW1vpPTZOr8L7RQimqai1a57bt5j22LzGZCw= -github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/wire v0.2.1 h1:TYj4Z2qjqxa2ufb34UJqVeO9aznL+i0fLO6TqThKZ7Y= github.com/google/wire v0.2.1/go.mod h1:ptBl5bWD3nzmJHVNwYHV3v4wdtKzBMlU2YbtKQCG9GI= -github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02 h1:Spo+4PFAGDqULAsZ7J69MOxq4/fwgZ0zvmDTBqpq7yU= -github.com/gorhill/cronexpr v0.0.0-20140423231348-a557574d6c02/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gosimple/slug v1.3.0 h1:NKQyQMjKkgCpD/Vd+wKtFc7N60bJNCLDubKU/UDKMFI= github.com/gosimple/slug v1.3.0/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0= -github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f h1:ShTPMJQes6tubcjzGMODIVG5hlrCeImaBnZzKF2N8SM= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hashicorp/consul v1.8.4 h1:XaCg97pt9APNhsaPAR6cd31psiWcO+cz+p9SBefnTtM= -github.com/hashicorp/consul/api v1.5.0 h1:Yo2bneoGy68A7aNwmuETFnPhjyBEm7n3vzRacEVMjvI= -github.com/hashicorp/consul/api v1.5.0/go.mod h1:LqwrLNW876eYSuUOo4ZLHBcdKc038txr/IMfbLPATa4= -github.com/hashicorp/consul/sdk v0.5.0 h1:WC4594Wp/LkEeML/OdQKEC1yqBmEYkRp6i7X5u0zDAs= -github.com/hashicorp/consul/sdk v0.5.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.14.1 h1:nQcJDQwIAGnmoUWp8ubocEX40cCml/17YkF6csQLReU= -github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= -github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= -github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-plugin v1.3.0 h1:4d/wJojzvHV1I4i/rrjVaeuyxWrLzDE1mDCyDy8fXS8= -github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= -github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= -github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= -github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= -github.com/hashicorp/memberlist v0.2.0 h1:WeeNspppWi5s1OFefTviPQueC/Bq8dONfvNjPhiEQKE= -github.com/hashicorp/memberlist v0.2.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/hashicorp/nomad v0.0.0-20190125003214-134391155854 h1:L7WhLZt2ory/kQWxqkMwOiBpIoa4BWoadN7yx8LHEtk= -github.com/hashicorp/nomad v0.0.0-20190125003214-134391155854/go.mod h1:WRaKjdO1G2iqi86TvTjIYtKTyxg4pl7NLr9InxtWaI0= -github.com/hashicorp/raft v1.1.2 h1:oxEL5DDeurYxLd3UbcY/hccgSPhLLpiBZ1YxtWEq59c= -github.com/hashicorp/raft v1.1.2/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8= -github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk= -github.com/hashicorp/serf v0.9.0 h1:+Zd/16AJ9lxk9RzfTDyv/TLhZ8UerqYS0/+JGCIDaa0= -github.com/hashicorp/serf v0.9.0/go.mod h1:YL0HO+FifKOW2u1ke99DGVu1zhcpZzNwrLIqBC7vbYU= -github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU= -github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= -github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8= -github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= -github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8 h1:12VvqtR6Aowv3l/EQUlocDHW2Cp4G9WJVH7uyH8QFJE= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= @@ -318,45 +184,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.1.0 h1:/5u4a+KGJptBRqGzPvYQL9p0d/tPR4S31+Tnzj9lEO4= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= -github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= -github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= -github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -364,30 +202,29 @@ github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 h1:dnMxwus89s86tI8r github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4/go.mod h1:cojhOHk1gbMeklOyDP2oKKLftefXoJreOQGOrXk+Z38= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -401,12 +238,6 @@ github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be h1:ta7tUOvsPHV github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be/go.mod h1:MIDFMn7db1kT65GmV94GzpX9Qdi7N/pQlwb+AN8wh+Q= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967 h1:x7xEyJDP7Hv3LVgvWhzioQqbC/KtuUhTigKlH/8ehhE= github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= -github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc= github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -414,34 +245,23 @@ github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606 h1:dU9yXzNi9rl6Mou7+3npdfPyeFPb2+7BHs3zL47bhPY= github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA= github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d h1:3wDi6J5APMqaHBVPuVd7RmHD2gRTfqbdcVSpCNoUWtk= github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.starlark.net v0.0.0-20201118183435-e55f603d8c79 h1:JPjLPz44y2N9mkzh2N344kTk1Y4/V4yJAjTrXGmzv8I= go.starlark.net v0.0.0-20201118183435-e55f603d8c79/go.mod h1:5YFcFnRptTN+41758c2bMPiqpGg4zBfYji1IQz8wNFk= golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -449,55 +269,46 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091 h1:DMyOG0U+gKfu8JZzg2UQe9MeaC1X+xQWlAKcRnjxjCw= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -509,64 +320,57 @@ golang.org/x/tools v0.0.0-20181017214349-06f26fdaaa28/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/h2non/gock.v1 v1.0.14 h1:fTeu9fcUvSnLNacYvYI54h+1/XEteDyHvrVCZEEEYNM= gopkg.in/h2non/gock.v1 v1.0.14/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= -gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.0.0-20181130031204-d04500c8c3dd h1:5aHsneN62ehs/tdtS9tWZlhVk68V7yms/Qw7nsGmvCA= k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20181204150028-eb8c8024849b h1:NBYMVxACHvRjnsH8rkNm2ICFZlXznkXYEefUdEpcueY= -k8s.io/apimachinery v0.0.0-20181204150028-eb8c8024849b/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/client-go v10.0.0+incompatible h1:F1IqCqw7oMBzDkqlcBymRq1450wD0eNqLE9jzUrIi34= -k8s.io/client-go v10.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= -k8s.io/klog v0.1.0 h1:I5HMfc/DtuVaGR1KPwUrTc476K8NCqNBldC7H4dYEzk= k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= From dc6a3ebebf928f364f33092033e417d6d10803b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Thu, 22 Jul 2021 15:11:42 +0200 Subject: [PATCH 3/8] fixed test --- pubsub/hub_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pubsub/hub_test.go b/pubsub/hub_test.go index ac728ab8..2c4c4e14 100644 --- a/pubsub/hub_test.go +++ b/pubsub/hub_test.go @@ -18,7 +18,7 @@ func TestBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - p := New() + p := newHub() events, errc := p.Subscribe(ctx) if got, want := p.Subscribers(), 1; got != want { From ff7013334168752e7c559137ffc3e1b1e2361730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Thu, 22 Jul 2021 18:56:41 +0200 Subject: [PATCH 4/8] added error as return value for pubsub.Subscribers() --- core/pubsub.go | 2 +- handler/api/system/stats.go | 8 +++++- mock/mock_gen.go | 5 ++-- pubsub/hub.go | 4 +-- pubsub/hub_test.go | 8 +++++- pubsub/redis.go | 54 ++++++++++++++++++++++++------------- 6 files changed, 56 insertions(+), 25 deletions(-) diff --git a/core/pubsub.go b/core/pubsub.go index c1e2d961..59e4d97a 100644 --- a/core/pubsub.go +++ b/core/pubsub.go @@ -33,5 +33,5 @@ type Pubsub interface { Subscribe(context.Context) (<-chan *Message, <-chan error) // Subscribers returns a count of subscribers. - Subscribers() int + Subscribers() (int, error) } diff --git a/handler/api/system/stats.go b/handler/api/system/stats.go index 4466a811..db5a429d 100644 --- a/handler/api/system/stats.go +++ b/handler/api/system/stats.go @@ -145,7 +145,13 @@ func HandleStats( // Event Stats // - stats.Events.Subscribers = bus.Subscribers() + stats.Events.Subscribers, err = bus.Subscribers() + if err != nil { + render.InternalError(w, err) + logger.FromRequest(r).WithError(err). + Warnln("stats: cannot get number of subscribers") + return + } // // Stream Stats diff --git a/mock/mock_gen.go b/mock/mock_gen.go index 73d3c297..b630a3dd 100644 --- a/mock/mock_gen.go +++ b/mock/mock_gen.go @@ -67,11 +67,12 @@ func (mr *MockPubsubMockRecorder) Subscribe(arg0 interface{}) *gomock.Call { } // Subscribers mocks base method. -func (m *MockPubsub) Subscribers() int { +func (m *MockPubsub) Subscribers() (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Subscribers") ret0, _ := ret[0].(int) - return ret0 + ret1, _ := ret[1].(error) + return ret0, ret1 } // Subscribers indicates an expected call of Subscribers. diff --git a/pubsub/hub.go b/pubsub/hub.go index 3445e7bc..42143974 100644 --- a/pubsub/hub.go +++ b/pubsub/hub.go @@ -65,9 +65,9 @@ func (h *hub) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan error return s.handler, errc } -func (h *hub) Subscribers() int { +func (h *hub) Subscribers() (int, error) { h.Lock() c := len(h.subs) h.Unlock() - return c + return c, nil } diff --git a/pubsub/hub_test.go b/pubsub/hub_test.go index 2c4c4e14..866eb888 100644 --- a/pubsub/hub_test.go +++ b/pubsub/hub_test.go @@ -21,7 +21,13 @@ func TestBus(t *testing.T) { p := newHub() events, errc := p.Subscribe(ctx) - if got, want := p.Subscribers(), 1; got != want { + got, err := p.Subscribers() + if err != nil { + t.Errorf("Test failed with an error: %s", err.Error()) + return + } + + if want := 1; got != want { t.Errorf("Want %d subscribers, got %d", want, got) } diff --git a/pubsub/redis.go b/pubsub/redis.go index c322b118..70d9af90 100644 --- a/pubsub/redis.go +++ b/pubsub/redis.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "os" "time" "github.com/drone/drone/core" @@ -68,18 +69,22 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan for { select { case m, ok := <-ch: - if ok { - message := &core.Message{} - err = json.Unmarshal([]byte(m.Payload), message) - if err != nil { - fmt.Printf("error@pubsub: failed to unmarshal a message. %s\n", err) - continue - } - messageCh <- message - } else { + if !ok { errCh <- fmt.Errorf("redis pubsub channel=%s closed", channelPubSub) return } + + message := &core.Message{} + err = json.Unmarshal([]byte(m.Payload), message) + if err != nil { + // This is a "should not happen" situation, + // because messages are encoded as json above in Publish(). + _, _ = fmt.Fprintf(os.Stderr, "error@pubsub: failed to unmarshal a message. %s\n", err) + continue + } + + messageCh <- message + case <-ctx.Done(): return } @@ -89,26 +94,39 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan return messageCh, errCh } -func (h *hubRedis) Subscribers() int { +func (h *hubRedis) Subscribers() (int, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFunc() v, err := h.rdb.Do(ctx, "pubsub", "numsub", channelPubSub).Result() if err != nil { - fmt.Printf("error@pubsub: failed to get number of subscribers. %s\n", err) - return -1 + err = fmt.Errorf("error@pubsub: failed to get number of subscribers. %w", err) + return 0, err } - values, ok := v.([]interface{}) // the result should be: [, ] + values, ok := v.([]interface{}) // the result should be: [, ] if !ok || len(values) != 2 { - return 0 + err = fmt.Errorf("error@pubsub: failed to extarct number of subscribers from: %v", values) + return 0, err } - if subscriberCount, ok := values[1].(int64); ok { - return int(subscriberCount) + switch n := values[1].(type) { + case int: + return n, nil + case uint: + return int(n), nil + case int32: + return int(n), nil + case uint32: + return int(n), nil + case int64: + return int(n), nil + case uint64: + return int(n), nil + default: + err = fmt.Errorf("error@pubsub: unsupported type for number of subscribers: %T", values[1]) + return 0, err } - - return 0 } func newRedis(rdb *redis.Client) (ps core.Pubsub, err error) { From b91d6c4b9bcc3992d53d22c472c0ce6abe2779e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Mon, 2 Aug 2021 13:30:52 +0200 Subject: [PATCH 5/8] added redis implemenatation for core.LogStream --- cmd/drone-server/inject_external.go | 51 +++++ cmd/drone-server/inject_store.go | 17 -- cmd/drone-server/wire.go | 1 + cmd/drone-server/wire_gen.go | 7 +- livelog/livelog.go | 81 +------ livelog/redis.go | 211 ++++++++++++++++++ livelog/streamer.go | 94 ++++++++ livelog/{livelog_test.go => streamer_test.go} | 12 +- pubsub/pubsub.go | 4 +- pubsub/redis.go | 30 +-- 10 files changed, 387 insertions(+), 121 deletions(-) create mode 100644 cmd/drone-server/inject_external.go create mode 100644 livelog/redis.go create mode 100644 livelog/streamer.go rename livelog/{livelog_test.go => streamer_test.go} (93%) diff --git a/cmd/drone-server/inject_external.go b/cmd/drone-server/inject_external.go new file mode 100644 index 00000000..515f1f90 --- /dev/null +++ b/cmd/drone-server/inject_external.go @@ -0,0 +1,51 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + + "github.com/drone/drone/cmd/drone-server/config" + + "github.com/go-redis/redis/v8" + "github.com/google/wire" +) + +// wire set for loading the external services. +var externalSet = wire.NewSet( + provideRedisClient, +) + +func provideRedisClient(config config.Config) (rdb *redis.Client, err error) { + if config.Redis.ConnectionString == "" { + return + } + + options, err := redis.ParseURL(config.Redis.ConnectionString) + if err != nil { + return + } + + rdb = redis.NewClient(options) + + _, err = rdb.Ping(context.Background()).Result() + if err != nil { + err = fmt.Errorf("redis not accessibe: %w", err) + return + } + + return +} diff --git a/cmd/drone-server/inject_store.go b/cmd/drone-server/inject_store.go index d9116856..0e06e9b0 100644 --- a/cmd/drone-server/inject_store.go +++ b/cmd/drone-server/inject_store.go @@ -34,7 +34,6 @@ import ( "github.com/drone/drone/store/template" "github.com/drone/drone/store/user" - "github.com/go-redis/redis/v8" "github.com/google/wire" "github.com/sirupsen/logrus" ) @@ -56,7 +55,6 @@ var storeSet = wire.NewSet( global.New, step.New, template.New, - provideRedisClient, ) // provideDatabase is a Wire provider function that provides a @@ -179,18 +177,3 @@ func provideUserStore(db *db.DB, enc encrypt.Encrypter, config config.Config) co metric.UserCount(users) return users } - -func provideRedisClient(config config.Config) (rdb *redis.Client, err error) { - if config.Redis.ConnectionString == "" { - return - } - - options, err := redis.ParseURL(config.Redis.ConnectionString) - if err != nil { - return - } - - rdb = redis.NewClient(options) - - return -} diff --git a/cmd/drone-server/wire.go b/cmd/drone-server/wire.go index 84aab8c6..bc937651 100644 --- a/cmd/drone-server/wire.go +++ b/cmd/drone-server/wire.go @@ -32,6 +32,7 @@ func InitializeApplication(config config.Config) (application, error) { serverSet, serviceSet, storeSet, + externalSet, newApplication, ) return application{}, nil diff --git a/cmd/drone-server/wire_gen.go b/cmd/drone-server/wire_gen.go index d65d65b7..ebc2f883 100644 --- a/cmd/drone-server/wire_gen.go +++ b/cmd/drone-server/wire_gen.go @@ -59,10 +59,7 @@ func InitializeApplication(config2 config.Config) (application, error) { if err != nil { return application{}, err } - corePubsub, err := pubsub.New(redisClient) - if err != nil { - return application{}, err - } + corePubsub := pubsub.New(redisClient) stageStore := provideStageStore(db) scheduler := provideScheduler(stageStore, config2) statusService := provideStatusService(client, renewer, config2) @@ -81,7 +78,7 @@ func InitializeApplication(config2 config.Config) (application, error) { coreLicense := provideLicense(client, config2) datadog := provideDatadog(userStore, repositoryStore, buildStore, system, coreLicense, config2) logStore := provideLogStore(db, config2) - logStream := livelog.New() + logStream := livelog.New(redisClient) netrcService := provideNetrcService(client, renewer, config2) secretStore := secret.New(db, encrypter) globalSecretStore := global.New(db, encrypter) diff --git a/livelog/livelog.go b/livelog/livelog.go index 9ab27d0b..6843732e 100644 --- a/livelog/livelog.go +++ b/livelog/livelog.go @@ -1,4 +1,4 @@ -// Copyright 2019 Drone IO, Inc. +// Copyright 2021 Drone IO, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,80 +15,17 @@ package livelog import ( - "context" - "errors" - "sync" - "github.com/drone/drone/core" + + "github.com/go-redis/redis/v8" ) -// error returned when a stream is not registered with -// the streamer. -var errStreamNotFound = errors.New("stream: not found") - -type streamer struct { - sync.Mutex - - streams map[int64]*stream -} - -// New returns a new in-memory log streamer. -func New() core.LogStream { - return &streamer{ - streams: make(map[int64]*stream), +// New creates a new log streamer. If Redis client passed as parameter is not nil it uses +// a Redis implementation, otherwise it uses an in-memory implementation. +func New(rdb *redis.Client) core.LogStream { + if rdb != nil { + return newRedis(rdb) } -} -func (s *streamer) Create(ctx context.Context, id int64) error { - s.Lock() - s.streams[id] = newStream() - s.Unlock() - return nil -} - -func (s *streamer) Delete(ctx context.Context, id int64) error { - s.Lock() - stream, ok := s.streams[id] - if ok { - delete(s.streams, id) - } - s.Unlock() - if !ok { - return errStreamNotFound - } - return stream.close() -} - -func (s *streamer) Write(ctx context.Context, id int64, line *core.Line) error { - s.Lock() - stream, ok := s.streams[id] - s.Unlock() - if !ok { - return errStreamNotFound - } - return stream.write(line) -} - -func (s *streamer) Tail(ctx context.Context, id int64) (<-chan *core.Line, <-chan error) { - s.Lock() - stream, ok := s.streams[id] - s.Unlock() - if !ok { - return nil, nil - } - return stream.subscribe(ctx) -} - -func (s *streamer) Info(ctx context.Context) *core.LogStreamInfo { - s.Lock() - defer s.Unlock() - info := &core.LogStreamInfo{ - Streams: map[int64]int{}, - } - for id, stream := range s.streams { - stream.Lock() - info.Streams[id] = len(stream.list) - stream.Unlock() - } - return info + return newStreamer() } diff --git a/livelog/redis.go b/livelog/redis.go new file mode 100644 index 00000000..5e7c670e --- /dev/null +++ b/livelog/redis.go @@ -0,0 +1,211 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package livelog + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/drone/drone/core" + + "github.com/go-redis/redis/v8" +) + +func newRedis(rdb *redis.Client) core.LogStream { + return &redisStream{ + client: rdb, + } +} + +const ( + redisKeyExpiryTime = 5 * time.Hour // How long each key exists in redis + redisPollTime = 100 * time.Millisecond // should not be too large to avoid redis clients getting occupied for long + redisTailMaxTime = 1 * time.Hour // maximum duration a tail can last + redisEntryKey = "line" + redisStreamPrefix = "drone-log-" +) + +type redisStream struct { + client redis.Cmdable +} + +// Create creates a redis stream and sets an expiry on it. +func (r *redisStream) Create(ctx context.Context, id int64) error { + // Delete if a stream already exists with the same key + _ = r.Delete(ctx, id) + + key := redisStreamPrefix + strconv.FormatInt(id, 10) + + addResp := r.client.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + ID: "*", // auto-generate a unique incremental ID + MaxLen: bufferSize, + Approx: true, + Values: map[string]interface{}{redisEntryKey: []byte{}}, + }) + if err := addResp.Err(); err != nil { + return fmt.Errorf("livelog/redis: could not create stream with key %s", key) + } + + res := r.client.Expire(ctx, key, redisKeyExpiryTime) + if err := res.Err(); err != nil { + return fmt.Errorf("livelog/redis: could not set expiry for key %s", key) + } + + return nil +} + +// Delete deletes a stream +func (r *redisStream) Delete(ctx context.Context, id int64) error { + key := redisStreamPrefix + strconv.FormatInt(id, 10) + + if err := r._exists(ctx, key); err != nil { + return err + } + + deleteResp := r.client.Del(ctx, key) + if err := deleteResp.Err(); err != nil { + return fmt.Errorf("livelog/redis: could not delete stream for step %d", id) + } + + return nil +} + +// Write writes information into the Redis stream +func (r *redisStream) Write(ctx context.Context, id int64, line *core.Line) error { + key := redisStreamPrefix + strconv.FormatInt(id, 10) + + if err := r._exists(ctx, key); err != nil { + return err + } + + lineJsonData, _ := json.Marshal(line) + addResp := r.client.XAdd(ctx, &redis.XAddArgs{ + Stream: key, + ID: "*", // auto-generate a unique incremental ID + MaxLen: bufferSize, + Approx: true, + Values: map[string]interface{}{redisEntryKey: lineJsonData}, + }) + if err := addResp.Err(); err != nil { + return err + } + + return nil +} + +// Tail returns back all the lines in the stream. +func (r *redisStream) Tail(ctx context.Context, id int64) (<-chan *core.Line, <-chan error) { + key := redisStreamPrefix + strconv.FormatInt(id, 10) + + if err := r._exists(ctx, key); err != nil { + return nil, nil + } + + chLines := make(chan *core.Line, bufferSize) + chErr := make(chan error, 1) + + go func() { + defer close(chErr) + defer close(chLines) + timeout := time.After(redisTailMaxTime) // polling should not last for longer than tailMaxTime + + // Keep reading from the stream and writing to the channel + lastID := "0" + + for { + select { + case <-ctx.Done(): + return + case <-timeout: + return + default: + readResp := r.client.XRead(ctx, &redis.XReadArgs{ + Streams: append([]string{key}, lastID), + Block: redisPollTime, // periodically check for ctx.Done + }) + if readResp.Err() != nil && readResp.Err() != redis.Nil { // readResp.Err() is sometimes set to "redis: nil" instead of nil + chErr <- readResp.Err() + return + } + + for _, msg := range readResp.Val() { + messages := msg.Messages + if len(messages) > 0 { + lastID = messages[len(messages)-1].ID + } else { // should not happen + return + } + + for _, message := range messages { + values := message.Values + if val, ok := values[redisEntryKey]; ok { + var line *core.Line + if err := json.Unmarshal([]byte(val.(string)), &line); err != nil { + continue // ignore errors in the stream + } + chLines <- line + } + } + } + } + } + }() + + return chLines, chErr +} + +// Info returns info about log streams present in redis +func (r *redisStream) Info(ctx context.Context) (info *core.LogStreamInfo) { + info = &core.LogStreamInfo{ + Streams: make(map[int64]int), + } + + keysResp := r.client.Keys(ctx, redisStreamPrefix+"*") + if err := keysResp.Err(); err != nil { + return + } + + for _, key := range keysResp.Val() { + ids := key[len(redisStreamPrefix):] + id, err := strconv.ParseInt(ids, 10, 64) + if err != nil { + continue + } + + lenResp := r.client.XLen(ctx, key) + if err := lenResp.Err(); err != nil { + continue + } + + size := int(lenResp.Val()) + + info.Streams[id] = size + } + + return +} + +func (r *redisStream) _exists(ctx context.Context, key string) error { + exists := r.client.Exists(ctx, key) + if exists.Err() != nil || exists.Val() == 0 { + return fmt.Errorf("livelog/redis: log stream %s not found", key) + } + + return nil +} diff --git a/livelog/streamer.go b/livelog/streamer.go new file mode 100644 index 00000000..09f0b166 --- /dev/null +++ b/livelog/streamer.go @@ -0,0 +1,94 @@ +// Copyright 2019 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package livelog + +import ( + "context" + "errors" + "sync" + + "github.com/drone/drone/core" +) + +// error returned when a stream is not registered with +// the streamer. +var errStreamNotFound = errors.New("stream: not found") + +type streamer struct { + sync.Mutex + + streams map[int64]*stream +} + +// New returns a new in-memory log streamer. +func newStreamer() core.LogStream { + return &streamer{ + streams: make(map[int64]*stream), + } +} + +func (s *streamer) Create(ctx context.Context, id int64) error { + s.Lock() + s.streams[id] = newStream() + s.Unlock() + return nil +} + +func (s *streamer) Delete(ctx context.Context, id int64) error { + s.Lock() + stream, ok := s.streams[id] + if ok { + delete(s.streams, id) + } + s.Unlock() + if !ok { + return errStreamNotFound + } + return stream.close() +} + +func (s *streamer) Write(ctx context.Context, id int64, line *core.Line) error { + s.Lock() + stream, ok := s.streams[id] + s.Unlock() + if !ok { + return errStreamNotFound + } + return stream.write(line) +} + +func (s *streamer) Tail(ctx context.Context, id int64) (<-chan *core.Line, <-chan error) { + s.Lock() + stream, ok := s.streams[id] + s.Unlock() + if !ok { + return nil, nil + } + return stream.subscribe(ctx) +} + +func (s *streamer) Info(ctx context.Context) *core.LogStreamInfo { + s.Lock() + defer s.Unlock() + info := &core.LogStreamInfo{ + Streams: map[int64]int{}, + } + for id, stream := range s.streams { + stream.Lock() + info.Streams[id] = len(stream.list) + stream.Unlock() + } + return info +} diff --git a/livelog/livelog_test.go b/livelog/streamer_test.go similarity index 93% rename from livelog/livelog_test.go rename to livelog/streamer_test.go index 4df79e51..df7f5f84 100644 --- a/livelog/livelog_test.go +++ b/livelog/streamer_test.go @@ -17,7 +17,7 @@ import ( ) func TestStreamer(t *testing.T) { - s := New().(*streamer) + s := newStreamer().(*streamer) err := s.Create(context.Background(), 1) if err != nil { t.Error(err) @@ -57,7 +57,7 @@ func TestStreamer(t *testing.T) { } func TestStreamerDelete(t *testing.T) { - s := New().(*streamer) + s := newStreamer().(*streamer) err := s.Create(context.Background(), 1) if err != nil { t.Error(err) @@ -75,7 +75,7 @@ func TestStreamerDelete(t *testing.T) { } func TestStreamerDeleteErr(t *testing.T) { - s := New() + s := newStreamer() err := s.Delete(context.Background(), 1) if err != errStreamNotFound { t.Errorf("Want errStreamNotFound") @@ -83,7 +83,7 @@ func TestStreamerDeleteErr(t *testing.T) { } func TestStreamerWriteErr(t *testing.T) { - s := New() + s := newStreamer() err := s.Write(context.Background(), 1, &core.Line{}) if err != errStreamNotFound { t.Errorf("Want errStreamNotFound") @@ -91,7 +91,7 @@ func TestStreamerWriteErr(t *testing.T) { } func TestStreamTailNotFound(t *testing.T) { - s := New() + s := newStreamer() outc, errc := s.Tail(context.Background(), 0) if outc != nil && errc != nil { t.Errorf("Expect nil channel when stream not found") @@ -99,7 +99,7 @@ func TestStreamTailNotFound(t *testing.T) { } func TestStreamerInfo(t *testing.T) { - s := New().(*streamer) + s := newStreamer().(*streamer) s.streams[1] = &stream{list: map[*subscriber]struct{}{{}: struct{}{}, {}: struct{}{}}} s.streams[2] = &stream{list: map[*subscriber]struct{}{{}: struct{}{}}} s.streams[3] = &stream{list: map[*subscriber]struct{}{}} diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 201f42e0..4c22b580 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -22,10 +22,10 @@ import ( // New creates a new publish subscriber. If Redis client passed as parameter is not nil it uses // a Redis implementation, otherwise it uses an in-memory implementation. -func New(rdb *redis.Client) (core.Pubsub, error) { +func New(rdb *redis.Client) core.Pubsub { if rdb != nil { return newRedis(rdb) } - return newHub(), nil + return newHub() } diff --git a/pubsub/redis.go b/pubsub/redis.go index 70d9af90..8a2b2b36 100644 --- a/pubsub/redis.go +++ b/pubsub/redis.go @@ -26,6 +26,12 @@ import ( "github.com/go-redis/redis/v8" ) +func newRedis(rdb *redis.Client) core.Pubsub { + return &hubRedis{ + rdb: rdb, + } +} + const channelPubSub = "drone-events" type hubRedis struct { @@ -70,7 +76,7 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan select { case m, ok := <-ch: if !ok { - errCh <- fmt.Errorf("redis pubsub channel=%s closed", channelPubSub) + errCh <- fmt.Errorf("pubsub/redis: channel=%s closed", channelPubSub) return } @@ -79,7 +85,7 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan if err != nil { // This is a "should not happen" situation, // because messages are encoded as json above in Publish(). - _, _ = fmt.Fprintf(os.Stderr, "error@pubsub: failed to unmarshal a message. %s\n", err) + _, _ = fmt.Fprintf(os.Stderr, "pubsub/redis: failed to unmarshal a message. %s\n", err) continue } @@ -100,13 +106,13 @@ func (h *hubRedis) Subscribers() (int, error) { v, err := h.rdb.Do(ctx, "pubsub", "numsub", channelPubSub).Result() if err != nil { - err = fmt.Errorf("error@pubsub: failed to get number of subscribers. %w", err) + err = fmt.Errorf("pubsub/redis: failed to get number of subscribers. %w", err) return 0, err } values, ok := v.([]interface{}) // the result should be: [, ] if !ok || len(values) != 2 { - err = fmt.Errorf("error@pubsub: failed to extarct number of subscribers from: %v", values) + err = fmt.Errorf("pubsub/redis: failed to extarct number of subscribers from: %v", values) return 0, err } @@ -124,21 +130,7 @@ func (h *hubRedis) Subscribers() (int, error) { case uint64: return int(n), nil default: - err = fmt.Errorf("error@pubsub: unsupported type for number of subscribers: %T", values[1]) + err = fmt.Errorf("pubsub/redis: unsupported type for number of subscribers: %T", values[1]) return 0, err } } - -func newRedis(rdb *redis.Client) (ps core.Pubsub, err error) { - _, err = rdb.Ping(context.Background()).Result() - if err != nil { - err = fmt.Errorf("redis not accessibe: %w", err) - return - } - - ps = &hubRedis{ - rdb: rdb, - } - - return -} From 3067a16de0b18a69c45d2552936e679bd77d072a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Mon, 2 Aug 2021 18:47:58 +0200 Subject: [PATCH 6/8] redis canceller --- cmd/drone-server/inject_scheduler.go | 6 +- cmd/drone-server/wire_gen.go | 2 +- pubsub/redis.go | 28 +++--- scheduler/queue/redis.go | 141 +++++++++++++++++++++++++++ scheduler/queue/scheduler.go | 24 ++++- 5 files changed, 179 insertions(+), 22 deletions(-) create mode 100644 scheduler/queue/redis.go diff --git a/cmd/drone-server/inject_scheduler.go b/cmd/drone-server/inject_scheduler.go index efc28d9f..fa3337a1 100644 --- a/cmd/drone-server/inject_scheduler.go +++ b/cmd/drone-server/inject_scheduler.go @@ -15,10 +15,10 @@ package main import ( - "github.com/drone/drone/cmd/drone-server/config" "github.com/drone/drone/core" "github.com/drone/drone/scheduler/queue" + "github.com/go-redis/redis/v8" "github.com/google/wire" ) @@ -29,6 +29,6 @@ var schedulerSet = wire.NewSet( // provideScheduler is a Wire provider function that returns a // scheduler based on the environment configuration. -func provideScheduler(store core.StageStore, config config.Config) core.Scheduler { - return queue.New(store) +func provideScheduler(store core.StageStore, rdb *redis.Client) core.Scheduler { + return queue.New(store, rdb) } diff --git a/cmd/drone-server/wire_gen.go b/cmd/drone-server/wire_gen.go index ebc2f883..295605f2 100644 --- a/cmd/drone-server/wire_gen.go +++ b/cmd/drone-server/wire_gen.go @@ -61,7 +61,7 @@ func InitializeApplication(config2 config.Config) (application, error) { } corePubsub := pubsub.New(redisClient) stageStore := provideStageStore(db) - scheduler := provideScheduler(stageStore, config2) + scheduler := provideScheduler(stageStore, redisClient) statusService := provideStatusService(client, renewer, config2) stepStore := step.New(db) system := provideSystem(config2) diff --git a/pubsub/redis.go b/pubsub/redis.go index 8a2b2b36..9fef153c 100644 --- a/pubsub/redis.go +++ b/pubsub/redis.go @@ -27,12 +27,10 @@ import ( ) func newRedis(rdb *redis.Client) core.Pubsub { - return &hubRedis{ - rdb: rdb, - } + return &hubRedis{rdb: rdb} } -const channelPubSub = "drone-events" +const redisPubSubEvents = "drone-events" type hubRedis struct { rdb *redis.Client @@ -44,7 +42,7 @@ func (h *hubRedis) Publish(ctx context.Context, e *core.Message) (err error) { return } - _, err = h.rdb.Publish(ctx, channelPubSub, data).Result() + _, err = h.rdb.Publish(ctx, redisPubSubEvents, data).Result() if err != nil { return } @@ -53,22 +51,22 @@ func (h *hubRedis) Publish(ctx context.Context, e *core.Message) (err error) { } func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan error) { - messageCh := make(chan *core.Message, 100) - errCh := make(chan error) + chMessage := make(chan *core.Message, 100) + chErr := make(chan error) go func() { - pubsub := h.rdb.Subscribe(ctx, channelPubSub) + pubsub := h.rdb.Subscribe(ctx, redisPubSubEvents) ch := pubsub.Channel(redis.WithChannelSize(100)) defer func() { _ = pubsub.Close() - close(messageCh) - close(errCh) + close(chMessage) + close(chErr) }() err := pubsub.Ping(ctx) if err != nil { - errCh <- err + chErr <- err return } @@ -76,7 +74,7 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan select { case m, ok := <-ch: if !ok { - errCh <- fmt.Errorf("pubsub/redis: channel=%s closed", channelPubSub) + chErr <- fmt.Errorf("pubsub/redis: channel=%s closed", redisPubSubEvents) return } @@ -89,7 +87,7 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan continue } - messageCh <- message + chMessage <- message case <-ctx.Done(): return @@ -97,14 +95,14 @@ func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan } }() - return messageCh, errCh + return chMessage, chErr } func (h *hubRedis) Subscribers() (int, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFunc() - v, err := h.rdb.Do(ctx, "pubsub", "numsub", channelPubSub).Result() + v, err := h.rdb.Do(ctx, "pubsub", "numsub", redisPubSubEvents).Result() if err != nil { err = fmt.Errorf("pubsub/redis: failed to get number of subscribers. %w", err) return 0, err diff --git a/scheduler/queue/redis.go b/scheduler/queue/redis.go new file mode 100644 index 00000000..f7507a64 --- /dev/null +++ b/scheduler/queue/redis.go @@ -0,0 +1,141 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package queue + +import ( + "context" + "fmt" + "os" + "strconv" + "time" + + "github.com/go-redis/redis/v8" +) + +const ( + redisPubSubCancel = "drone-cancel" + redisCancelValuePrefix = "drone-cancel-" + redisCancelValueTimeout = 5 * time.Minute + redisCancelValue = "canceled" +) + +func newRedisCanceller(rdb *redis.Client) *redisCanceller { + return &redisCanceller{rdb: rdb} +} + +type redisCanceller struct { + rdb *redis.Client +} + +func (c *redisCanceller) Cancel(ctx context.Context, id int64) (err error) { + ids := strconv.FormatInt(id, 10) + + // publish a cancel event to all subscribers (runners) waiting to + _, err = c.rdb.Publish(ctx, redisPubSubCancel, ids).Result() + if err != nil { + return + } + + // put a limited duration value in case a runner isn't listening currently. + _, err = c.rdb.Set(ctx, redisCancelValuePrefix+ids, redisCancelValue, redisCancelValueTimeout).Result() + if err != nil { + return + } + + return nil +} + +func (c *redisCanceller) Cancelled(ctx context.Context, id int64) (isCancelled bool, err error) { + ids := strconv.FormatInt(id, 10) + + // first check if the build is already cancelled + + result, err := c.rdb.Get(ctx, redisCancelValuePrefix+ids).Result() + if err != nil && err != redis.Nil { + fmt.Println("GOT AN ERROR", err.Error()) ////////////////////////////////////////////////////// + return + } + + isCancelled = err != redis.Nil && result == redisCancelValue + if isCancelled { + fmt.Println("BUILD IS CANCELED", result) //////////////////////////////////////////////// + return + } + + // if it is not cancelled, subscribe and listen to cancel build events + // until the context is cancelled or until the build is cancelled. + + chResult := make(chan interface{}) + + go func() { + pubsub := c.rdb.Subscribe(ctx, redisPubSubCancel) + ch := pubsub.Channel() + + defer func() { + _ = pubsub.Close() + close(chResult) + }() + + err := pubsub.Ping(ctx) + if err != nil { + chResult <- err + return + } + + for { + select { + case m, ok := <-ch: + if !ok { + chResult <- fmt.Errorf("canceller/redis: channel=%s closed", redisPubSubCancel) + return + } + + idMessage, err := strconv.ParseInt(m.Payload, 10, 64) + if err != nil { // should not happen + _, _ = fmt.Fprintf(os.Stderr, "canceller/redis: message is not an integer: %s\n", m.Payload) + continue // ignore data errors + } + + if id == idMessage { + fmt.Println("BUILD HAS JUST BEEN CANCELED", m.Payload) //////////////////////////////////////////////// + chResult <- true + } else { ////////////////////// + fmt.Println("SOMETHING ELSE IS CANCELLED", m.Payload) //////////////////////////////////////////////// + } ///////////////////////////// + + case <-ctx.Done(): + return + } + } + }() + + value, ok := <-chResult + + if !ok { + return + } + + err, ok = value.(error) + if ok { + return + } + + isCancelled, ok = value.(bool) + if ok { + return + } + + return +} diff --git a/scheduler/queue/scheduler.go b/scheduler/queue/scheduler.go index d0a4c9d9..2ab5c5de 100644 --- a/scheduler/queue/scheduler.go +++ b/scheduler/queue/scheduler.go @@ -19,6 +19,8 @@ import ( "errors" "github.com/drone/drone/core" + + "github.com/go-redis/redis/v8" ) type scheduler struct { @@ -26,14 +28,30 @@ type scheduler struct { *canceller } +type redisScheduler struct { + *queue + *redisCanceller +} + // New creates a new scheduler. -func New(store core.StageStore) core.Scheduler { - return &scheduler{ +func New(store core.StageStore, rdb *redis.Client) core.Scheduler { + if rdb != nil { + return redisScheduler{ + queue: newQueue(store), + redisCanceller: newRedisCanceller(rdb), + } + } + + return scheduler{ queue: newQueue(store), canceller: newCanceller(), } } -func (d *scheduler) Stats(context.Context) (interface{}, error) { +func (d scheduler) Stats(context.Context) (interface{}, error) { + return nil, errors.New("not implemented") +} + +func (d redisScheduler) Stats(context.Context) (interface{}, error) { return nil, errors.New("not implemented") } From fb2963660849936bda7e65212f7460cc5b4e7d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Mon, 9 Aug 2021 11:56:33 +0200 Subject: [PATCH 7/8] h/a stuff is !oss, simplified redis pubsub subscribe --- cmd/drone-server/inject_external.go | 26 +---- cmd/drone-server/inject_scheduler.go | 6 +- cmd/drone-server/wire_gen.go | 8 +- livelog/livelog.go | 9 +- livelog/livelogOSS.go | 27 +++++ livelog/{redis.go => streamRedis.go} | 53 +++++---- pubsub/hubRedis.go | 124 +++++++++++++++++++++ pubsub/pubsub.go | 11 +- pubsub/pubsubOSS.go | 27 +++++ pubsub/redis.go | 134 ---------------------- scheduler/queue/cancellerRedis.go | 160 ++++++++++++++++++++++++++ scheduler/queue/redis.go | 136 ---------------------- scheduler/queue/scheduler.go | 30 +---- scheduler/queue/schedulerNonOSS.go | 37 ++++++ scheduler/queue/schedulerOSS.go | 30 +++++ scheduler/queue/schedulerRedis.go | 31 ++++++ service/redisdb/redisdb.go | 161 +++++++++++++++++++++++++++ 17 files changed, 653 insertions(+), 357 deletions(-) create mode 100644 livelog/livelogOSS.go rename livelog/{redis.go => streamRedis.go} (79%) create mode 100644 pubsub/hubRedis.go create mode 100644 pubsub/pubsubOSS.go delete mode 100644 pubsub/redis.go create mode 100644 scheduler/queue/cancellerRedis.go delete mode 100644 scheduler/queue/redis.go create mode 100644 scheduler/queue/schedulerNonOSS.go create mode 100644 scheduler/queue/schedulerOSS.go create mode 100644 scheduler/queue/schedulerRedis.go create mode 100644 service/redisdb/redisdb.go diff --git a/cmd/drone-server/inject_external.go b/cmd/drone-server/inject_external.go index 515f1f90..d5a29a1e 100644 --- a/cmd/drone-server/inject_external.go +++ b/cmd/drone-server/inject_external.go @@ -15,12 +15,9 @@ package main import ( - "context" - "fmt" - "github.com/drone/drone/cmd/drone-server/config" + "github.com/drone/drone/service/redisdb" - "github.com/go-redis/redis/v8" "github.com/google/wire" ) @@ -29,23 +26,6 @@ var externalSet = wire.NewSet( provideRedisClient, ) -func provideRedisClient(config config.Config) (rdb *redis.Client, err error) { - if config.Redis.ConnectionString == "" { - return - } - - options, err := redis.ParseURL(config.Redis.ConnectionString) - if err != nil { - return - } - - rdb = redis.NewClient(options) - - _, err = rdb.Ping(context.Background()).Result() - if err != nil { - err = fmt.Errorf("redis not accessibe: %w", err) - return - } - - return +func provideRedisClient(config config.Config) (rdb redisdb.RedisDB, err error) { + return redisdb.New(config) } diff --git a/cmd/drone-server/inject_scheduler.go b/cmd/drone-server/inject_scheduler.go index fa3337a1..42a6053c 100644 --- a/cmd/drone-server/inject_scheduler.go +++ b/cmd/drone-server/inject_scheduler.go @@ -17,8 +17,8 @@ package main import ( "github.com/drone/drone/core" "github.com/drone/drone/scheduler/queue" + "github.com/drone/drone/service/redisdb" - "github.com/go-redis/redis/v8" "github.com/google/wire" ) @@ -29,6 +29,6 @@ var schedulerSet = wire.NewSet( // provideScheduler is a Wire provider function that returns a // scheduler based on the environment configuration. -func provideScheduler(store core.StageStore, rdb *redis.Client) core.Scheduler { - return queue.New(store, rdb) +func provideScheduler(store core.StageStore, r redisdb.RedisDB) core.Scheduler { + return queue.New(store, r) } diff --git a/cmd/drone-server/wire_gen.go b/cmd/drone-server/wire_gen.go index 295605f2..33783fe5 100644 --- a/cmd/drone-server/wire_gen.go +++ b/cmd/drone-server/wire_gen.go @@ -55,13 +55,13 @@ func InitializeApplication(config2 config.Config) (application, error) { cronStore := cron.New(db) repositoryStore := provideRepoStore(db) buildStore := provideBuildStore(db) - redisClient, err := provideRedisClient(config2) + redisDB, err := provideRedisClient(config2) if err != nil { return application{}, err } - corePubsub := pubsub.New(redisClient) + corePubsub := pubsub.New(redisDB) stageStore := provideStageStore(db) - scheduler := provideScheduler(stageStore, redisClient) + scheduler := provideScheduler(stageStore, redisDB) statusService := provideStatusService(client, renewer, config2) stepStore := step.New(db) system := provideSystem(config2) @@ -78,7 +78,7 @@ func InitializeApplication(config2 config.Config) (application, error) { coreLicense := provideLicense(client, config2) datadog := provideDatadog(userStore, repositoryStore, buildStore, system, coreLicense, config2) logStore := provideLogStore(db, config2) - logStream := livelog.New(redisClient) + logStream := livelog.New(redisDB) netrcService := provideNetrcService(client, renewer, config2) secretStore := secret.New(db, encrypter) globalSecretStore := global.New(db, encrypter) diff --git a/livelog/livelog.go b/livelog/livelog.go index 6843732e..5e288189 100644 --- a/livelog/livelog.go +++ b/livelog/livelog.go @@ -12,19 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !oss + package livelog import ( "github.com/drone/drone/core" - - "github.com/go-redis/redis/v8" + "github.com/drone/drone/service/redisdb" ) // New creates a new log streamer. If Redis client passed as parameter is not nil it uses // a Redis implementation, otherwise it uses an in-memory implementation. -func New(rdb *redis.Client) core.LogStream { +func New(rdb redisdb.RedisDB) core.LogStream { if rdb != nil { - return newRedis(rdb) + return newStreamRedis(rdb) } return newStreamer() diff --git a/livelog/livelogOSS.go b/livelog/livelogOSS.go new file mode 100644 index 00000000..c638b25e --- /dev/null +++ b/livelog/livelogOSS.go @@ -0,0 +1,27 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build oss + +package livelog + +import ( + "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" +) + +// New creates a new in-memory log streamer. +func New(r redisdb.RedisDB) core.LogStream { + return newStreamer() +} diff --git a/livelog/redis.go b/livelog/streamRedis.go similarity index 79% rename from livelog/redis.go rename to livelog/streamRedis.go index 5e7c670e..9a72fab7 100644 --- a/livelog/redis.go +++ b/livelog/streamRedis.go @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !oss + package livelog import ( @@ -22,13 +24,14 @@ import ( "time" "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" "github.com/go-redis/redis/v8" ) -func newRedis(rdb *redis.Client) core.LogStream { - return &redisStream{ - client: rdb, +func newStreamRedis(r redisdb.RedisDB) core.LogStream { + return streamRedis{ + rdb: r, } } @@ -40,18 +43,20 @@ const ( redisStreamPrefix = "drone-log-" ) -type redisStream struct { - client redis.Cmdable +type streamRedis struct { + rdb redisdb.RedisDB } // Create creates a redis stream and sets an expiry on it. -func (r *redisStream) Create(ctx context.Context, id int64) error { +func (r streamRedis) Create(ctx context.Context, id int64) error { // Delete if a stream already exists with the same key _ = r.Delete(ctx, id) + client := r.rdb.Client() + key := redisStreamPrefix + strconv.FormatInt(id, 10) - addResp := r.client.XAdd(ctx, &redis.XAddArgs{ + addResp := client.XAdd(ctx, &redis.XAddArgs{ Stream: key, ID: "*", // auto-generate a unique incremental ID MaxLen: bufferSize, @@ -62,7 +67,7 @@ func (r *redisStream) Create(ctx context.Context, id int64) error { return fmt.Errorf("livelog/redis: could not create stream with key %s", key) } - res := r.client.Expire(ctx, key, redisKeyExpiryTime) + res := client.Expire(ctx, key, redisKeyExpiryTime) if err := res.Err(); err != nil { return fmt.Errorf("livelog/redis: could not set expiry for key %s", key) } @@ -71,14 +76,16 @@ func (r *redisStream) Create(ctx context.Context, id int64) error { } // Delete deletes a stream -func (r *redisStream) Delete(ctx context.Context, id int64) error { +func (r streamRedis) Delete(ctx context.Context, id int64) error { + client := r.rdb.Client() + key := redisStreamPrefix + strconv.FormatInt(id, 10) if err := r._exists(ctx, key); err != nil { return err } - deleteResp := r.client.Del(ctx, key) + deleteResp := client.Del(ctx, key) if err := deleteResp.Err(); err != nil { return fmt.Errorf("livelog/redis: could not delete stream for step %d", id) } @@ -87,7 +94,9 @@ func (r *redisStream) Delete(ctx context.Context, id int64) error { } // Write writes information into the Redis stream -func (r *redisStream) Write(ctx context.Context, id int64, line *core.Line) error { +func (r streamRedis) Write(ctx context.Context, id int64, line *core.Line) error { + client := r.rdb.Client() + key := redisStreamPrefix + strconv.FormatInt(id, 10) if err := r._exists(ctx, key); err != nil { @@ -95,7 +104,7 @@ func (r *redisStream) Write(ctx context.Context, id int64, line *core.Line) erro } lineJsonData, _ := json.Marshal(line) - addResp := r.client.XAdd(ctx, &redis.XAddArgs{ + addResp := client.XAdd(ctx, &redis.XAddArgs{ Stream: key, ID: "*", // auto-generate a unique incremental ID MaxLen: bufferSize, @@ -110,7 +119,9 @@ func (r *redisStream) Write(ctx context.Context, id int64, line *core.Line) erro } // Tail returns back all the lines in the stream. -func (r *redisStream) Tail(ctx context.Context, id int64) (<-chan *core.Line, <-chan error) { +func (r streamRedis) Tail(ctx context.Context, id int64) (<-chan *core.Line, <-chan error) { + client := r.rdb.Client() + key := redisStreamPrefix + strconv.FormatInt(id, 10) if err := r._exists(ctx, key); err != nil { @@ -135,7 +146,7 @@ func (r *redisStream) Tail(ctx context.Context, id int64) (<-chan *core.Line, <- case <-timeout: return default: - readResp := r.client.XRead(ctx, &redis.XReadArgs{ + readResp := client.XRead(ctx, &redis.XReadArgs{ Streams: append([]string{key}, lastID), Block: redisPollTime, // periodically check for ctx.Done }) @@ -171,12 +182,14 @@ func (r *redisStream) Tail(ctx context.Context, id int64) (<-chan *core.Line, <- } // Info returns info about log streams present in redis -func (r *redisStream) Info(ctx context.Context) (info *core.LogStreamInfo) { +func (r streamRedis) Info(ctx context.Context) (info *core.LogStreamInfo) { + client := r.rdb.Client() + info = &core.LogStreamInfo{ Streams: make(map[int64]int), } - keysResp := r.client.Keys(ctx, redisStreamPrefix+"*") + keysResp := client.Keys(ctx, redisStreamPrefix+"*") if err := keysResp.Err(); err != nil { return } @@ -188,7 +201,7 @@ func (r *redisStream) Info(ctx context.Context) (info *core.LogStreamInfo) { continue } - lenResp := r.client.XLen(ctx, key) + lenResp := client.XLen(ctx, key) if err := lenResp.Err(); err != nil { continue } @@ -201,8 +214,10 @@ func (r *redisStream) Info(ctx context.Context) (info *core.LogStreamInfo) { return } -func (r *redisStream) _exists(ctx context.Context, key string) error { - exists := r.client.Exists(ctx, key) +func (r streamRedis) _exists(ctx context.Context, key string) error { + client := r.rdb.Client() + + exists := client.Exists(ctx, key) if exists.Err() != nil || exists.Val() == 0 { return fmt.Errorf("livelog/redis: log stream %s not found", key) } diff --git a/pubsub/hubRedis.go b/pubsub/hubRedis.go new file mode 100644 index 00000000..8b546d6e --- /dev/null +++ b/pubsub/hubRedis.go @@ -0,0 +1,124 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !oss + +package pubsub + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sync" + + "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" +) + +const ( + redisPubSubEvents = "drone-events" + redisPubSubCapacity = 100 +) + +func newHubRedis(r redisdb.RedisDB) core.Pubsub { + h := &hubRedis{ + rdb: r, + subscribers: make(map[chan<- *core.Message]struct{}), + } + + go r.Subscribe(context.Background(), redisPubSubEvents, redisPubSubCapacity, h) + + return h +} + +type hubRedis struct { + sync.Mutex + rdb redisdb.RedisDB + subscribers map[chan<- *core.Message]struct{} +} + +// Publish publishes a new message. All subscribers will get it. +func (h *hubRedis) Publish(ctx context.Context, e *core.Message) (err error) { + client := h.rdb.Client() + + data, err := json.Marshal(e) + if err != nil { + return + } + + _, err = client.Publish(ctx, redisPubSubEvents, data).Result() + if err != nil { + return + } + + return +} + +// Subscribe add a new subscriber. The subscriber gets event until its context is not finished. +func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan error) { + chMessage := make(chan *core.Message, redisPubSubCapacity) + chErr := make(chan error) + + h.Lock() + h.subscribers[chMessage] = struct{}{} + h.Unlock() + + go func() { + <-ctx.Done() + + h.Lock() + delete(h.subscribers, chMessage) + h.Unlock() + + close(chMessage) + close(chErr) + }() + + return chMessage, chErr +} + +// Subscribers returns number of subscribers. +func (h *hubRedis) Subscribers() (int, error) { + h.Lock() + n := len(h.subscribers) + h.Unlock() + + return n, nil +} + +// ProcessMessage relays the message to all subscribers listening to drone events. +// It is a part of redisdb.PubSubProcessor implementation and it's called internally by redisdb.Subscribe. +func (h *hubRedis) ProcessMessage(s string) { + message := &core.Message{} + err := json.Unmarshal([]byte(s), message) + if err != nil { + // Ignore invalid messages. This is a "should not happen" situation, + // because messages are encoded as json in Publish(). + _, _ = fmt.Fprintf(os.Stderr, "pubsub/redis: failed to unmarshal a message. %s\n", err) + return + } + + h.Lock() + for ss := range h.subscribers { + select { + case ss <- message: + default: // messages are lost if a subscriber channel reaches its capacity + } + } + h.Unlock() +} + +// ProcessError is a part of redisdb.PubSubProcessor implementation. +func (h *hubRedis) ProcessError(error) {} diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 4c22b580..f4f2db16 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -12,19 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +// +build !oss + package pubsub import ( "github.com/drone/drone/core" - - "github.com/go-redis/redis/v8" + "github.com/drone/drone/service/redisdb" ) // New creates a new publish subscriber. If Redis client passed as parameter is not nil it uses // a Redis implementation, otherwise it uses an in-memory implementation. -func New(rdb *redis.Client) core.Pubsub { - if rdb != nil { - return newRedis(rdb) +func New(r redisdb.RedisDB) core.Pubsub { + if r != nil { + return newHubRedis(r) } return newHub() diff --git a/pubsub/pubsubOSS.go b/pubsub/pubsubOSS.go new file mode 100644 index 00000000..85a01fd6 --- /dev/null +++ b/pubsub/pubsubOSS.go @@ -0,0 +1,27 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build oss + +package pubsub + +import ( + "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" +) + +// New creates a new in-memory publish subscriber. +func New(r redisdb.RedisDB) core.Pubsub { + return newHub() +} diff --git a/pubsub/redis.go b/pubsub/redis.go deleted file mode 100644 index 9fef153c..00000000 --- a/pubsub/redis.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2021 Drone IO, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pubsub - -import ( - "context" - "encoding/json" - "fmt" - "os" - "time" - - "github.com/drone/drone/core" - - "github.com/go-redis/redis/v8" -) - -func newRedis(rdb *redis.Client) core.Pubsub { - return &hubRedis{rdb: rdb} -} - -const redisPubSubEvents = "drone-events" - -type hubRedis struct { - rdb *redis.Client -} - -func (h *hubRedis) Publish(ctx context.Context, e *core.Message) (err error) { - data, err := json.Marshal(e) - if err != nil { - return - } - - _, err = h.rdb.Publish(ctx, redisPubSubEvents, data).Result() - if err != nil { - return - } - - return -} - -func (h *hubRedis) Subscribe(ctx context.Context) (<-chan *core.Message, <-chan error) { - chMessage := make(chan *core.Message, 100) - chErr := make(chan error) - - go func() { - pubsub := h.rdb.Subscribe(ctx, redisPubSubEvents) - ch := pubsub.Channel(redis.WithChannelSize(100)) - - defer func() { - _ = pubsub.Close() - close(chMessage) - close(chErr) - }() - - err := pubsub.Ping(ctx) - if err != nil { - chErr <- err - return - } - - for { - select { - case m, ok := <-ch: - if !ok { - chErr <- fmt.Errorf("pubsub/redis: channel=%s closed", redisPubSubEvents) - return - } - - message := &core.Message{} - err = json.Unmarshal([]byte(m.Payload), message) - if err != nil { - // This is a "should not happen" situation, - // because messages are encoded as json above in Publish(). - _, _ = fmt.Fprintf(os.Stderr, "pubsub/redis: failed to unmarshal a message. %s\n", err) - continue - } - - chMessage <- message - - case <-ctx.Done(): - return - } - } - }() - - return chMessage, chErr -} - -func (h *hubRedis) Subscribers() (int, error) { - ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) - defer cancelFunc() - - v, err := h.rdb.Do(ctx, "pubsub", "numsub", redisPubSubEvents).Result() - if err != nil { - err = fmt.Errorf("pubsub/redis: failed to get number of subscribers. %w", err) - return 0, err - } - - values, ok := v.([]interface{}) // the result should be: [, ] - if !ok || len(values) != 2 { - err = fmt.Errorf("pubsub/redis: failed to extarct number of subscribers from: %v", values) - return 0, err - } - - switch n := values[1].(type) { - case int: - return n, nil - case uint: - return int(n), nil - case int32: - return int(n), nil - case uint32: - return int(n), nil - case int64: - return int(n), nil - case uint64: - return int(n), nil - default: - err = fmt.Errorf("pubsub/redis: unsupported type for number of subscribers: %T", values[1]) - return 0, err - } -} diff --git a/scheduler/queue/cancellerRedis.go b/scheduler/queue/cancellerRedis.go new file mode 100644 index 00000000..d43a976f --- /dev/null +++ b/scheduler/queue/cancellerRedis.go @@ -0,0 +1,160 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !oss + +package queue + +import ( + "context" + "fmt" + "os" + "strconv" + "sync" + "time" + + "github.com/drone/drone/service/redisdb" + + "github.com/go-redis/redis/v8" +) + +const ( + redisPubSubCancel = "drone-cancel" + redisCancelValuePrefix = "drone-cancel-" + redisCancelValueTimeout = 5 * time.Minute + redisCancelValue = "canceled" +) + +func newCancellerRedis(r redisdb.RedisDB) *cancellerRedis { + h := &cancellerRedis{ + rdb: r, + subscribers: make(map[*cancelSubscriber]struct{}), + } + + go r.Subscribe(context.Background(), redisPubSubCancel, 1, h) + + return h +} + +type cancellerRedis struct { + rdb redisdb.RedisDB + subscribers map[*cancelSubscriber]struct{} + sync.Mutex +} + +type cancelSubscriber struct { + id int64 + ch chan<- error +} + +// Cancel informs all subscribers that a build with the provided id is cancelled. +func (c *cancellerRedis) Cancel(ctx context.Context, id int64) (err error) { + client := c.rdb.Client() + + ids := strconv.FormatInt(id, 10) + + // publish a cancel event to all subscribers (runners) waiting to + _, err = client.Publish(ctx, redisPubSubCancel, ids).Result() + if err != nil { + return + } + + // put a limited duration value in case a runner isn't listening currently. + _, err = client.Set(ctx, redisCancelValuePrefix+ids, redisCancelValue, redisCancelValueTimeout).Result() + if err != nil { + return + } + + return +} + +// Cancelled waits until it gets info that a build with the provided id is cancelled. +// The waiting is aborted when the provided context is done. +func (c *cancellerRedis) Cancelled(ctx context.Context, id int64) (isCancelled bool, err error) { + client := c.rdb.Client() + + ids := strconv.FormatInt(id, 10) + + // first check if the build is already cancelled + + result, err := client.Get(ctx, redisCancelValuePrefix+ids).Result() + if err != nil && err != redis.Nil { + return + } + + isCancelled = err != redis.Nil && result == redisCancelValue + if isCancelled { + return + } + + // if it is not cancelled, subscribe and listen to cancel build events + // until the context is cancelled or until the build is cancelled. + + ch := make(chan error) + sub := &cancelSubscriber{id: id, ch: ch} + + c.Lock() + c.subscribers[sub] = struct{}{} + c.Unlock() + + select { + case err = <-ch: + // If the build is cancelled or an error happened, + // than the subscriber is removed from the set by other go routine + isCancelled = err != nil + case <-ctx.Done(): + // If the context is cancelled then the subscriber must be be removed here. + c.Lock() + delete(c.subscribers, sub) + c.Unlock() + } + + return +} + +// ProcessMessage informs all subscribers listening to cancellation that the build with this id is cancelled. +// It is a part of redisdb.PubSubProcessor implementation and it's called internally by Subscribe. +func (c *cancellerRedis) ProcessMessage(s string) { + id, err := strconv.ParseInt(s, 10, 64) + if err != nil { + // Ignore invalid messages. This is a "should not happen" situation, + // because all messages are integers as strings in method Cancel(). + _, _ = fmt.Fprintf(os.Stderr, "canceller/redis: message is not an integer: %s\n", s) + return + } + + c.Lock() + for ss := range c.subscribers { + if ss.id == id { + ss.ch <- nil + close(ss.ch) + delete(c.subscribers, ss) + } + } + c.Unlock() +} + +// ProcessError informs all subscribers that an error happened and clears the set of subscribers. +// The set of subscribers is cleared because each subscriber receives only one message, +// so an error could cause that the message is missed - it's safer to return an error. +// It is a part of redisdb.PubSubProcessor implementation and it's called internally by Subscribe. +func (c *cancellerRedis) ProcessError(err error) { + c.Lock() + for ss := range c.subscribers { + ss.ch <- err + close(ss.ch) + delete(c.subscribers, ss) + } + c.Unlock() +} diff --git a/scheduler/queue/redis.go b/scheduler/queue/redis.go deleted file mode 100644 index f5096a87..00000000 --- a/scheduler/queue/redis.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2021 Drone IO, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package queue - -import ( - "context" - "fmt" - "os" - "strconv" - "time" - - "github.com/go-redis/redis/v8" -) - -const ( - redisPubSubCancel = "drone-cancel" - redisCancelValuePrefix = "drone-cancel-" - redisCancelValueTimeout = 5 * time.Minute - redisCancelValue = "canceled" -) - -func newRedisCanceller(rdb *redis.Client) *redisCanceller { - return &redisCanceller{rdb: rdb} -} - -type redisCanceller struct { - rdb *redis.Client -} - -func (c *redisCanceller) Cancel(ctx context.Context, id int64) (err error) { - ids := strconv.FormatInt(id, 10) - - // publish a cancel event to all subscribers (runners) waiting to - _, err = c.rdb.Publish(ctx, redisPubSubCancel, ids).Result() - if err != nil { - return - } - - // put a limited duration value in case a runner isn't listening currently. - _, err = c.rdb.Set(ctx, redisCancelValuePrefix+ids, redisCancelValue, redisCancelValueTimeout).Result() - if err != nil { - return - } - - return nil -} - -func (c *redisCanceller) Cancelled(ctx context.Context, id int64) (isCancelled bool, err error) { - ids := strconv.FormatInt(id, 10) - - // first check if the build is already cancelled - - result, err := c.rdb.Get(ctx, redisCancelValuePrefix+ids).Result() - if err != nil && err != redis.Nil { - return - } - - isCancelled = err != redis.Nil && result == redisCancelValue - if isCancelled { - return - } - - // if it is not cancelled, subscribe and listen to cancel build events - // until the context is cancelled or until the build is cancelled. - - chResult := make(chan interface{}) - - go func() { - pubsub := c.rdb.Subscribe(ctx, redisPubSubCancel) - ch := pubsub.Channel() - - defer func() { - _ = pubsub.Close() - close(chResult) - }() - - err := pubsub.Ping(ctx) - if err != nil { - chResult <- err - return - } - - for { - select { - case m, ok := <-ch: - if !ok { - chResult <- fmt.Errorf("canceller/redis: channel=%s closed", redisPubSubCancel) - return - } - - idMessage, err := strconv.ParseInt(m.Payload, 10, 64) - if err != nil { // should not happen - _, _ = fmt.Fprintf(os.Stderr, "canceller/redis: message is not an integer: %s\n", m.Payload) - continue // ignore data errors - } - - if id == idMessage { - chResult <- true - } - - case <-ctx.Done(): - return - } - } - }() - - value, ok := <-chResult - - if !ok { - return - } - - err, ok = value.(error) - if ok { - return - } - - isCancelled, ok = value.(bool) - if ok { - return - } - - return -} diff --git a/scheduler/queue/scheduler.go b/scheduler/queue/scheduler.go index 2ab5c5de..44620870 100644 --- a/scheduler/queue/scheduler.go +++ b/scheduler/queue/scheduler.go @@ -1,4 +1,4 @@ -// Copyright 2019 Drone IO, Inc. +// Copyright 2021 Drone IO, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -17,10 +17,6 @@ package queue import ( "context" "errors" - - "github.com/drone/drone/core" - - "github.com/go-redis/redis/v8" ) type scheduler struct { @@ -28,30 +24,6 @@ type scheduler struct { *canceller } -type redisScheduler struct { - *queue - *redisCanceller -} - -// New creates a new scheduler. -func New(store core.StageStore, rdb *redis.Client) core.Scheduler { - if rdb != nil { - return redisScheduler{ - queue: newQueue(store), - redisCanceller: newRedisCanceller(rdb), - } - } - - return scheduler{ - queue: newQueue(store), - canceller: newCanceller(), - } -} - func (d scheduler) Stats(context.Context) (interface{}, error) { return nil, errors.New("not implemented") } - -func (d redisScheduler) Stats(context.Context) (interface{}, error) { - return nil, errors.New("not implemented") -} diff --git a/scheduler/queue/schedulerNonOSS.go b/scheduler/queue/schedulerNonOSS.go new file mode 100644 index 00000000..d23fb6c1 --- /dev/null +++ b/scheduler/queue/schedulerNonOSS.go @@ -0,0 +1,37 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !oss + +package queue + +import ( + "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" +) + +// New creates a new scheduler. +func New(store core.StageStore, r redisdb.RedisDB) core.Scheduler { + if r == nil { + return scheduler{ + queue: newQueue(store), + canceller: newCanceller(), + } + } + + return schedulerRedis{ + queue: newQueue(store), + cancellerRedis: newCancellerRedis(r), + } +} diff --git a/scheduler/queue/schedulerOSS.go b/scheduler/queue/schedulerOSS.go new file mode 100644 index 00000000..5cee23bb --- /dev/null +++ b/scheduler/queue/schedulerOSS.go @@ -0,0 +1,30 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build oss + +package queue + +import ( + "github.com/drone/drone/core" + "github.com/drone/drone/service/redisdb" +) + +// New creates a new scheduler. +func New(store core.StageStore, r redisdb.RedisDB) core.Scheduler { + return scheduler{ + queue: newQueue(store), + canceller: newCanceller(), + } +} diff --git a/scheduler/queue/schedulerRedis.go b/scheduler/queue/schedulerRedis.go new file mode 100644 index 00000000..a14cf489 --- /dev/null +++ b/scheduler/queue/schedulerRedis.go @@ -0,0 +1,31 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !oss + +package queue + +import ( + "context" + "errors" +) + +type schedulerRedis struct { + *queue + *cancellerRedis +} + +func (d schedulerRedis) Stats(context.Context) (interface{}, error) { + return nil, errors.New("not implemented") +} diff --git a/service/redisdb/redisdb.go b/service/redisdb/redisdb.go new file mode 100644 index 00000000..f604229c --- /dev/null +++ b/service/redisdb/redisdb.go @@ -0,0 +1,161 @@ +// Copyright 2021 Drone IO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package redisdb + +import ( + "context" + "fmt" + "time" + + "github.com/drone/drone/cmd/drone-server/config" + + "github.com/go-redis/redis/v8" + "github.com/sirupsen/logrus" +) + +func New(config config.Config) (srv RedisDB, err error) { + if config.Redis.ConnectionString == "" { + return + } + + options, err := redis.ParseURL(config.Redis.ConnectionString) + if err != nil { + return + } + + rdb := redis.NewClient(options) + + _, err = rdb.Ping(context.Background()).Result() + if err != nil { + err = fmt.Errorf("redis not accessibe: %w", err) + return + } + + srv = redisService{ + rdb: rdb, + } + + return +} + +type RedisDB interface { + Client() redis.Cmdable + Subscribe(ctx context.Context, channelName string, channelSize int, proc PubSubProcessor) +} + +type redisService struct { + rdb *redis.Client +} + +// Client exposes redis.Cmdable interface +func (r redisService) Client() redis.Cmdable { + return r.rdb +} + +type PubSubProcessor interface { + ProcessMessage(s string) + ProcessError(err error) +} + +var backoffDurations = []time.Duration{ + 0, time.Second, 3 * time.Second, 5 * time.Second, 10 * time.Second, 20 * time.Second, +} + +// Subscribe subscribes to a redis pub-sub channel. The messages are processed with the supplied PubSubProcessor. +// In case of en error the function will automatically reconnect with an increasing back of delay. +// The only way to exit this function is to terminate or expire the supplied context. +func (r redisService) Subscribe(ctx context.Context, channelName string, channelSize int, proc PubSubProcessor) { + var connectTry int + for { + err := func() (err error) { + defer func() { + // panic recovery because external PubSubProcessor methods might cause panics. + if p := recover(); p != nil { + err = fmt.Errorf("redis pubsub: panic: %v", p) + } + }() + + var options []redis.ChannelOption + + if channelSize > 1 { + options = append(options, redis.WithChannelSize(channelSize)) + } + + pubsub := r.rdb.Subscribe(ctx, channelName) + ch := pubsub.Channel(options...) + + defer func() { + _ = pubsub.Close() + }() + + // make sure the connection is successful + err = pubsub.Ping(ctx) + if err != nil { + return + } + + connectTry = 0 // successfully connected, reset the counter + + logrus. + WithField("try", connectTry+1). + WithField("channel", channelName). + Trace("redis pubsub: subscribed") + + for { + select { + case m, ok := <-ch: + if !ok { + err = fmt.Errorf("redis pubsub: channel=%s closed", channelName) + return + } + + proc.ProcessMessage(m.Payload) + + case <-ctx.Done(): + err = ctx.Err() + return + } + } + }() + if err == nil { + // should not happen, the function should always exit with an error + continue + } + + proc.ProcessError(err) + + if err == context.Canceled || err == context.DeadlineExceeded { + logrus. + WithField("channel", channelName). + Trace("redis pubsub: finished") + return + } + + dur := backoffDurations[connectTry] + + logrus. + WithError(err). + WithField("try", connectTry+1). + WithField("pause", dur.String()). + WithField("channel", channelName). + Error("redis pubsub: connection failed, reconnecting") + + time.Sleep(dur) + + if connectTry < len(backoffDurations)-1 { + connectTry++ + } + } +} From cf7d30a7f6d63f461fc7637c2b6c0c35fb5053e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Wed, 11 Aug 2021 16:25:50 +0200 Subject: [PATCH 8/8] added more redis config env vars --- cmd/drone-server/config/config.go | 3 +++ service/redisdb/redisdb.go | 18 +++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/cmd/drone-server/config/config.go b/cmd/drone-server/config/config.go index 29b7eae0..94f52ad4 100644 --- a/cmd/drone-server/config/config.go +++ b/cmd/drone-server/config/config.go @@ -170,6 +170,9 @@ type ( // Redis provides the redis configuration. Redis struct { ConnectionString string `envconfig:"DRONE_REDIS_CONNECTION"` + Addr string `envconfig:"DRONE_REDIS_ADDR"` + Password string `envconfig:"DRONE_REDIS_PASSWORD"` + DB int `envconfig:"DRONE_REDIS_DB"` } // Repository provides the repository configuration. diff --git a/service/redisdb/redisdb.go b/service/redisdb/redisdb.go index f604229c..7024cb67 100644 --- a/service/redisdb/redisdb.go +++ b/service/redisdb/redisdb.go @@ -26,12 +26,20 @@ import ( ) func New(config config.Config) (srv RedisDB, err error) { - if config.Redis.ConnectionString == "" { - return - } + var options *redis.Options - options, err := redis.ParseURL(config.Redis.ConnectionString) - if err != nil { + if config.Redis.ConnectionString != "" { + options, err = redis.ParseURL(config.Redis.ConnectionString) + if err != nil { + return + } + } else if config.Redis.Addr != "" { + options = &redis.Options{ + Addr: config.Redis.Addr, + Password: config.Redis.Password, + DB: config.Redis.DB, + } + } else { return }