Merge pull request #2788 from Lucretius/azure-blob-logs
Add support for primary store of Azure blob
This commit is contained in:
Brad Rydzewski
GitHub
commit
d581dab302
6 changed files with 159 additions and 10 deletions
|
|
@ -49,6 +49,7 @@ type (
|
|||
|
||||
Authn Authentication
|
||||
Agent Agent
|
||||
AzureBlob AzureBlob
|
||||
Cron Cron
|
||||
Cloning Cloning
|
||||
Database Database
|
||||
|
|
@ -362,6 +363,13 @@ type (
|
|||
PathStyle bool `envconfig:"DRONE_S3_PATH_STYLE"`
|
||||
}
|
||||
|
||||
//AzureBlob providers the storage configuration.
|
||||
AzureBlob struct {
|
||||
ContainerName string `envconfig:"DRONE_AZURE_BLOB_CONTAINER_NAME"`
|
||||
StorageAccountName string `envconfig:"DRONE_AZURE_STORAGE_ACCOUNT_NAME"`
|
||||
StorageAccessKey string `envconfig:"DRONE_AZURE_STORAGE_ACCESS_KEY"`
|
||||
}
|
||||
|
||||
// HTTP provides http configuration.
|
||||
HTTP struct {
|
||||
AllowedHosts []string `envconfig:"DRONE_HTTP_ALLOWED_HOSTS"`
|
||||
|
|
|
|||
|
|
@ -81,17 +81,25 @@ func provideBuildStore(db *db.DB) core.BuildStore {
|
|||
// provideLogStore is a Wire provider function that provides a
|
||||
// log datastore, configured from the environment.
|
||||
func provideLogStore(db *db.DB, config config.Config) core.LogStore {
|
||||
if config.S3.Bucket == "" {
|
||||
return logs.New(db)
|
||||
}
|
||||
s := logs.New(db)
|
||||
p := logs.NewS3Env(
|
||||
config.S3.Bucket,
|
||||
config.S3.Prefix,
|
||||
config.S3.Endpoint,
|
||||
config.S3.PathStyle,
|
||||
)
|
||||
return logs.NewCombined(p, s)
|
||||
if config.S3.Bucket != "" {
|
||||
p := logs.NewS3Env(
|
||||
config.S3.Bucket,
|
||||
config.S3.Prefix,
|
||||
config.S3.Endpoint,
|
||||
config.S3.PathStyle,
|
||||
)
|
||||
return logs.NewCombined(p, s)
|
||||
}
|
||||
if config.AzureBlob.ContainerName != "" {
|
||||
p := logs.NewAzureBlobEnv(
|
||||
config.AzureBlob.ContainerName,
|
||||
config.AzureBlob.StorageAccountName,
|
||||
config.AzureBlob.StorageAccessKey,
|
||||
)
|
||||
return logs.NewCombined(p, s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// provideStageStore is a Wire provider function that provides a
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -3,6 +3,7 @@ module github.com/drone/drone
|
|||
require (
|
||||
docker.io/go-docker v1.0.0
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e
|
||||
github.com/Azure/azure-storage-blob-go v0.7.0
|
||||
github.com/Microsoft/go-winio v0.4.11
|
||||
github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f
|
||||
github.com/aws/aws-sdk-go v1.15.57
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -2,6 +2,10 @@ docker.io/go-docker v1.0.0 h1:VdXS/aNYQxyA9wdLD5z8Q8Ro688/hG8HzKxYVEVbE6s=
|
|||
docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY=
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc=
|
||||
github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY=
|
||||
github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
|
||||
github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
|
||||
github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck=
|
||||
github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
|
||||
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
|
|
@ -178,6 +182,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
|
|||
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
|
||||
github.com/hashicorp/go-retryablehttp v0.0.0-20180718195005-e651d75abec6 h1:qCv4319q2q7XKn0MQbi8p37hsJ+9Xo8e6yojA73JVxk=
|
||||
github.com/hashicorp/go-retryablehttp v0.0.0-20180718195005-e651d75abec6/go.mod h1:fXcdFsQoipQa7mwORhKad5jmDCeSy/RCGzWA08PO0lM=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
|
||||
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
|
||||
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/nomad v0.0.0-20190125003214-134391155854 h1:L7WhLZt2ory/kQWxqkMwOiBpIoa4BWoadN7yx8LHEtk=
|
||||
|
|
@ -200,6 +206,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
|
||||
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA=
|
||||
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
|
||||
github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=
|
||||
|
|
|
|||
100
store/logs/azureblob.go
Normal file
100
store/logs/azureblob.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
// Copyright 2019 Drone.IO Inc. All rights reserved.
|
||||
// Use of this source code is governed by the Drone Non-Commercial License
|
||||
// that can be found in the LICENSE file.
|
||||
|
||||
// +build !oss
|
||||
|
||||
package logs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"github.com/drone/drone/core"
|
||||
)
|
||||
|
||||
// NewAzureBlobEnv returns a new Azure blob log store.
|
||||
func NewAzureBlobEnv(containerName, storageAccountName, storageAccessKey string) core.LogStore {
|
||||
return &azureBlobStore{
|
||||
containerName: containerName,
|
||||
storageAccountName: storageAccountName,
|
||||
storageAccessKey: storageAccessKey,
|
||||
containerURL: nil,
|
||||
}
|
||||
}
|
||||
|
||||
type azureBlobStore struct {
|
||||
containerName string
|
||||
storageAccountName string
|
||||
storageAccessKey string
|
||||
containerURL *azblob.ContainerURL
|
||||
}
|
||||
|
||||
func (az *azureBlobStore) Find(ctx context.Context, step int64) (io.ReadCloser, error) {
|
||||
err := az.getContainerURL()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
|
||||
out, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Body(azblob.RetryReaderOptions{}), nil
|
||||
}
|
||||
|
||||
func (az *azureBlobStore) Create(ctx context.Context, step int64, r io.Reader) error {
|
||||
err := az.getContainerURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts := &azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: 4 * 1024 * 1024,
|
||||
MaxBuffers: 5,
|
||||
}
|
||||
blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
|
||||
_, err = azblob.UploadStreamToBlockBlob(ctx, r, blobURL, *opts)
|
||||
return err
|
||||
}
|
||||
|
||||
func (az *azureBlobStore) Update(ctx context.Context, step int64, r io.Reader) error {
|
||||
return az.Create(ctx, step, r)
|
||||
}
|
||||
|
||||
func (az *azureBlobStore) Delete(ctx context.Context, step int64) error {
|
||||
err := az.getContainerURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
|
||||
_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func (az *azureBlobStore) getContainerURL() error {
|
||||
if az.containerURL != nil {
|
||||
return nil
|
||||
}
|
||||
if len(az.storageAccountName) == 0 || len(az.storageAccessKey) == 0 {
|
||||
return fmt.Errorf("Either the storage account or storage access key environment variable is not set")
|
||||
}
|
||||
credential, err := azblob.NewSharedKeyCredential(az.storageAccountName, az.storageAccessKey)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
|
||||
URL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", az.storageAccountName, az.containerName))
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
containerURL := azblob.NewContainerURL(*URL, p)
|
||||
az.containerURL = &containerURL
|
||||
return nil
|
||||
}
|
||||
24
store/logs/azureblob_oss.go
Normal file
24
store/logs/azureblob_oss.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
// Copyright 2019 Drone IO, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// +build oss
|
||||
|
||||
package logs
|
||||
|
||||
import "github.com/drone/drone/core"
|
||||
|
||||
// New returns a zero value LogStore.
|
||||
func NewAzureBlobEnv(containerName, storageAccountName, storageAccessKey string) core.LogStore {
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in a new issue