Merge pull request #2788 from Lucretius/azure-blob-logs

Add support for primary store of Azure blob

cmd/drone-server/config/config.go

@@ -49,6 +49,7 @@ type (
 
 		Authn        Authentication
 		Agent        Agent
+		AzureBlob    AzureBlob
 		Cron         Cron
 		Cloning      Cloning
 		Database     Database
@@ -362,6 +363,13 @@ type (
 		PathStyle bool   `envconfig:"DRONE_S3_PATH_STYLE"`
 	}
 
+	//AzureBlob providers the storage configuration.
+	AzureBlob struct {
+		ContainerName      string `envconfig:"DRONE_AZURE_BLOB_CONTAINER_NAME"`
+		StorageAccountName string `envconfig:"DRONE_AZURE_STORAGE_ACCOUNT_NAME"`
+		StorageAccessKey   string `envconfig:"DRONE_AZURE_STORAGE_ACCESS_KEY"`
+	}
+
 	// HTTP provides http configuration.
 	HTTP struct {
 		AllowedHosts          []string          `envconfig:"DRONE_HTTP_ALLOWED_HOSTS"`

cmd/drone-server/inject_store.go

@@ -81,17 +81,25 @@ func provideBuildStore(db *db.DB) core.BuildStore {
 // provideLogStore is a Wire provider function that provides a
 // log datastore, configured from the environment.
 func provideLogStore(db *db.DB, config config.Config) core.LogStore {
-	if config.S3.Bucket == "" {
-		return logs.New(db)
-	}
 	s := logs.New(db)
-	p := logs.NewS3Env(
+	if config.S3.Bucket != "" {
-		config.S3.Bucket,
+		p := logs.NewS3Env(
-		config.S3.Prefix,
+			config.S3.Bucket,
-		config.S3.Endpoint,
+			config.S3.Prefix,
-		config.S3.PathStyle,
+			config.S3.Endpoint,
-	)
+			config.S3.PathStyle,
-	return logs.NewCombined(p, s)
+		)
+		return logs.NewCombined(p, s)
+	}
+	if config.AzureBlob.ContainerName != "" {
+		p := logs.NewAzureBlobEnv(
+			config.AzureBlob.ContainerName,
+			config.AzureBlob.StorageAccountName,
+			config.AzureBlob.StorageAccessKey,
+		)
+		return logs.NewCombined(p, s)
+	}
+	return s
 }
 
 // provideStageStore is a Wire provider function that provides a

go.mod

@@ -3,6 +3,7 @@ module github.com/drone/drone
 require (
 	docker.io/go-docker v1.0.0
 	github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e
+	github.com/Azure/azure-storage-blob-go v0.7.0
 	github.com/Microsoft/go-winio v0.4.11
 	github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f
 	github.com/aws/aws-sdk-go v1.15.57

go.sum

@@ -2,6 +2,10 @@ docker.io/go-docker v1.0.0 h1:VdXS/aNYQxyA9wdLD5z8Q8Ro688/hG8HzKxYVEVbE6s=
 docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY=
 github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc=
 github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY=
+github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo=
+github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
+github.com/Azure/azure-storage-blob-go v0.7.0 h1:MuueVOYkufCxJw5YZzF842DY2MBsp+hLuh2apKY0mck=
+github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4=
 github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -178,6 +182,8 @@ github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uP
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-retryablehttp v0.0.0-20180718195005-e651d75abec6 h1:qCv4319q2q7XKn0MQbi8p37hsJ+9Xo8e6yojA73JVxk=
 github.com/hashicorp/go-retryablehttp v0.0.0-20180718195005-e651d75abec6/go.mod h1:fXcdFsQoipQa7mwORhKad5jmDCeSy/RCGzWA08PO0lM=
+github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
 github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/nomad v0.0.0-20190125003214-134391155854 h1:L7WhLZt2ory/kQWxqkMwOiBpIoa4BWoadN7yx8LHEtk=
@@ -200,6 +206,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA=
+github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
 github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4=

store/logs/azureblob.go

@@ -0,0 +1,100 @@
+// Copyright 2019 Drone.IO Inc. All rights reserved.
+// Use of this source code is governed by the Drone Non-Commercial License
+// that can be found in the LICENSE file.
+
+// +build !oss
+
+package logs
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/drone/drone/core"
+)
+
+// NewAzureBlobEnv returns a new Azure blob log store.
+func NewAzureBlobEnv(containerName, storageAccountName, storageAccessKey string) core.LogStore {
+	return &azureBlobStore{
+		containerName:      containerName,
+		storageAccountName: storageAccountName,
+		storageAccessKey:   storageAccessKey,
+		containerURL:       nil,
+	}
+}
+
+type azureBlobStore struct {
+	containerName      string
+	storageAccountName string
+	storageAccessKey   string
+	containerURL       *azblob.ContainerURL
+}
+
+func (az *azureBlobStore) Find(ctx context.Context, step int64) (io.ReadCloser, error) {
+	err := az.getContainerURL()
+	if err != nil {
+		return nil, err
+	}
+	blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
+	out, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
+	if err != nil {
+		return nil, err
+	}
+	return out.Body(azblob.RetryReaderOptions{}), nil
+}
+
+func (az *azureBlobStore) Create(ctx context.Context, step int64, r io.Reader) error {
+	err := az.getContainerURL()
+	if err != nil {
+		return err
+	}
+	opts := &azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 5,
+	}
+	blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
+	_, err = azblob.UploadStreamToBlockBlob(ctx, r, blobURL, *opts)
+	return err
+}
+
+func (az *azureBlobStore) Update(ctx context.Context, step int64, r io.Reader) error {
+	return az.Create(ctx, step, r)
+}
+
+func (az *azureBlobStore) Delete(ctx context.Context, step int64) error {
+	err := az.getContainerURL()
+	if err != nil {
+		return err
+	}
+	blobURL := az.containerURL.NewBlockBlobURL(fmt.Sprintf("%d", step))
+	_, err = blobURL.Delete(ctx, azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{})
+	return err
+}
+
+func (az *azureBlobStore) getContainerURL() error {
+	if az.containerURL != nil {
+		return nil
+	}
+	if len(az.storageAccountName) == 0 || len(az.storageAccessKey) == 0 {
+		return fmt.Errorf("Either the storage account or storage access key environment variable is not set")
+	}
+	credential, err := azblob.NewSharedKeyCredential(az.storageAccountName, az.storageAccessKey)
+
+	if err != nil {
+		return err
+	}
+
+	p := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+	URL, err := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/%s", az.storageAccountName, az.containerName))
+
+	if err != nil {
+		return err
+	}
+
+	containerURL := azblob.NewContainerURL(*URL, p)
+	az.containerURL = &containerURL
+	return nil
+}

store/logs/azureblob_oss.go

@@ -0,0 +1,24 @@
+// Copyright 2019 Drone IO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build oss
+
+package logs
+
+import "github.com/drone/drone/core"
+
+// New returns a zero value LogStore.
+func NewAzureBlobEnv(containerName, storageAccountName, storageAccessKey string) core.LogStore {
+	return nil
+}