Switch to AWS SDK for S3 cache access

The Minio AWS library doesn't support a number of items, such as:

* S3 Express One Zone
* The ability to configure AWS_STS_ENDPOINT_URL for AWS Secret Cloud

This new S3 client can be toggled off via the
FF_USE_LEGACY_S3_CACHE_ADAPTER feature flag.

Relates to https://gitlab.com/gitlab-org/gitlab-runner/-/issues/37394

Changelog: changed

Author: stanhu

cache/cache.go

@@ -70,6 +70,10 @@ func getAdaptorForBuild(build *common.Build, key string) Adapter {
 		build.Runner.Cache.Type = "gcsv2"
 	}
 
+	if build.Runner.Cache.Type == "s3" && !build.IsFeatureFlagOn(featureflags.UseLegacyS3CacheAdapter) {
+		build.Runner.Cache.Type = "s3v2"
+	}
+
 	adapter, err := createAdapter(build.Runner.Cache, build.GetBuildTimeout(), objectName)
 	if err != nil {
 		logrus.WithError(err).Error("Could not create cache adapter")

cache/s3v2/adapter.go

@@ -0,0 +1,95 @@
+package s3v2
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/sirupsen/logrus"
+
+	"gitlab.com/gitlab-org/gitlab-runner/cache"
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+type s3Adapter struct {
+	timeout    time.Duration
+	config     *common.CacheS3Config
+	objectName string
+	client     s3Presigner
+}
+
+func (a *s3Adapter) GetDownloadURL(ctx context.Context) *url.URL {
+	presignedURL, err := a.presignURL(ctx, http.MethodGet)
+	if err != nil {
+		logrus.WithError(err).Error("error while generating S3 pre-signed URL")
+
+		return nil
+	}
+
+	return presignedURL
+}
+
+func (a *s3Adapter) GetUploadURL(ctx context.Context) *url.URL {
+	presignedURL, err := a.presignURL(ctx, http.MethodPut)
+	if err != nil {
+		logrus.WithError(err).Error("error while generating S3 pre-signed URL")
+
+		return nil
+	}
+
+	return presignedURL
+}
+
+func (a *s3Adapter) GetUploadHeaders() http.Header {
+	return nil
+}
+
+func (a *s3Adapter) GetGoCloudURL(_ context.Context) *url.URL {
+	return nil
+}
+
+func (a *s3Adapter) GetUploadEnv() map[string]string {
+	return nil
+}
+
+func (a *s3Adapter) presignURL(ctx context.Context, method string) (*url.URL, error) {
+	if a.config.BucketName == "" {
+		return nil, fmt.Errorf("config BucketName cannot be empty")
+	}
+
+	if a.objectName == "" {
+		return nil, fmt.Errorf("object name cannot be empty")
+	}
+
+	return a.client.PresignURL(ctx, method, a.config.BucketName, a.objectName, a.timeout)
+}
+
+func New(config *common.CacheConfig, timeout time.Duration, objectName string) (cache.Adapter, error) {
+	s3Config := config.S3
+	if s3Config == nil {
+		return nil, fmt.Errorf("missing S3 configuration")
+	}
+
+	client, err := newS3Client(s3Config)
+	if err != nil {
+		return nil, fmt.Errorf("error while creating S3 cache storage client: %w", err)
+	}
+
+	a := &s3Adapter{
+		config:     s3Config,
+		timeout:    timeout,
+		objectName: objectName,
+		client:     client,
+	}
+
+	return a, nil
+}
+
+func init() {
+	err := cache.Factories().Register("s3v2", New)
+	if err != nil {
+		panic(err)
+	}
+}

cache/s3v2/adapter_test.go

@@ -0,0 +1,154 @@
+//go:build !integration
+
+package s3v2
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"gitlab.com/gitlab-org/gitlab-runner/cache"
+	"gitlab.com/gitlab-org/gitlab-runner/common"
+)
+
+var defaultTimeout = 1 * time.Hour
+
+const (
+	bucketName     = "test"
+	objectName     = "key"
+	bucketLocation = "location"
+)
+
+func defaultCacheFactory() *common.CacheConfig {
+	return &common.CacheConfig{
+		Type: "s3v2",
+		S3: &common.CacheS3Config{
+			ServerAddress:  "server.com",
+			AccessKey:      "access",
+			SecretKey:      "key",
+			BucketName:     bucketName,
+			BucketLocation: bucketLocation},
+	}
+}
+
+type cacheOperationTest struct {
+	errorOnS3ClientInitialization bool
+	errorOnURLPresigning          bool
+
+	presignedURL *url.URL
+	expectedURL  *url.URL
+}
+
+func onFakeS3URLGenerator(tc cacheOperationTest) func() {
+	client := new(mockS3Presigner)
+
+	var err error
+	if tc.errorOnURLPresigning {
+		err = errors.New("test error")
+	}
+
+	client.
+		On(
+			"PresignURL", mock.Anything, mock.Anything, mock.Anything,
+			mock.Anything, mock.Anything,
+		).
+		Return(tc.presignedURL, err)
+
+	oldS3URLGenerator := newS3Client
+	newS3Client = func(s3 *common.CacheS3Config) (s3Presigner, error) {
+		if tc.errorOnS3ClientInitialization {
+			return nil, errors.New("test error")
+		}
+		return client, nil
+	}
+
+	return func() {
+		newS3Client = oldS3URLGenerator
+	}
+}
+
+func testCacheOperation(
+	t *testing.T,
+	operationName string,
+	operation func(adapter cache.Adapter) *url.URL,
+	tc cacheOperationTest,
+	cacheConfig *common.CacheConfig,
+) {
+	t.Run(operationName, func(t *testing.T) {
+		cleanupS3URLGeneratorMock := onFakeS3URLGenerator(tc)
+		defer cleanupS3URLGeneratorMock()
+
+		adapter, err := New(cacheConfig, defaultTimeout, objectName)
+
+		if tc.errorOnS3ClientInitialization {
+			assert.EqualError(t, err, "error while creating S3 cache storage client: test error")
+
+			return
+		}
+		require.NoError(t, err)
+
+		URL := operation(adapter)
+		assert.Equal(t, tc.expectedURL, URL)
+
+		uploadHeaders := adapter.GetUploadHeaders()
+		assert.Nil(t, uploadHeaders)
+
+		assert.Nil(t, adapter.GetGoCloudURL(context.Background()))
+		assert.Empty(t, adapter.GetUploadEnv())
+	})
+}
+
+func TestCacheOperation(t *testing.T) {
+	URL, err := url.Parse("https://s3.example.com")
+	require.NoError(t, err)
+
+	tests := map[string]cacheOperationTest{
+		"error-on-s3-client-initialization": {
+			errorOnS3ClientInitialization: true,
+		},
+		"error-on-presigning-url": {
+			errorOnURLPresigning: true,
+			presignedURL:         URL,
+			expectedURL:          nil,
+		},
+		"presigned-url": {
+			presignedURL: URL,
+			expectedURL:  URL,
+		},
+	}
+
+	for testName, test := range tests {
+		t.Run(testName, func(t *testing.T) {
+			testCacheOperation(
+				t,
+				"GetDownloadURL",
+				func(adapter cache.Adapter) *url.URL { return adapter.GetDownloadURL(context.Background()) },
+				test,
+				defaultCacheFactory(),
+			)
+			testCacheOperation(
+				t,
+				"GetUploadURL",
+				func(adapter cache.Adapter) *url.URL { return adapter.GetUploadURL(context.Background()) },
+				test,
+				defaultCacheFactory(),
+			)
+		})
+	}
+}
+
+func TestNoConfiguration(t *testing.T) {
+	s3Cache := defaultCacheFactory()
+	s3Cache.S3 = nil
+
+	adapter, err := New(s3Cache, defaultTimeout, objectName)
+	assert.Nil(t, adapter)
+
+	assert.EqualError(t, err, "missing S3 configuration")
+}