Skip to content

Commit 033e00a

Browse files
wojaswojas
authored
Merge pull request #46 from ahouene/minio-k8s-credentials
S3 Backend: load keys from file with automatic reload
2 parents a0dd21b + 0c624b3 commit 033e00a

File tree

5 files changed

+344
-6
lines changed

5 files changed

+344
-6
lines changed

‎backends/s3/credentials.go‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package s3
2+
3+
import (
4+
"os"
5+
"time"
6+
7+
"github.com/minio/minio-go/v7/pkg/credentials"
8+
)
9+
10+
// FileSecretsCredentials is an implementation of Minio's credentials.Provider,
11+
// allowing to read credentials from Kubernetes or Docker secrets, as described in
12+
// https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure
13+
// and https://docs.docker.com/engine/swarm/secrets.
14+
type FileSecretsCredentials struct {
15+
credentials.Expiry
16+
17+
// Path to the file containing the access key,
18+
// e.g. /etc/s3-secrets/access-key.
19+
AccessKeyFile string
20+
21+
// Path to the file containing the secret key,
22+
// e.g. /etc/s3-secrets/secret-key.
23+
SecretKeyFile string
24+
25+
// Time between each secrets retrieval.
26+
RefreshInterval time.Duration
27+
}
28+
29+
// Retrieve implements credentials.Provider.
30+
// It reads files pointed to by p.AccessKeyFilename and p.SecretKeyFilename.
31+
func (c *FileSecretsCredentials) Retrieve() (credentials.Value, error) {
32+
keyId, err := os.ReadFile(c.AccessKeyFile)
33+
if err != nil {
34+
return credentials.Value{}, err
35+
}
36+
secretKey, err := os.ReadFile(c.SecretKeyFile)
37+
if err != nil {
38+
return credentials.Value{}, err
39+
}
40+
41+
creds := credentials.Value{
42+
AccessKeyID: string(keyId),
43+
SecretAccessKey: string(secretKey),
44+
}
45+
46+
c.SetExpiration(time.Now().Add(c.RefreshInterval), -1)
47+
48+
return creds, err
49+
}
50+
51+
var _ credentials.Provider = new(FileSecretsCredentials)

‎backends/s3/credentials_test.go‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package s3_test
2+
3+
import (
4+
"context"
5+
"errors"
6+
"os"
7+
"path/filepath"
8+
"testing"
9+
"time"
10+
11+
"github.com/PowerDNS/simpleblob/backends/s3"
12+
"github.com/PowerDNS/simpleblob/backends/s3/s3testing"
13+
"github.com/PowerDNS/simpleblob/tester"
14+
"github.com/minio/minio-go/v7"
15+
"github.com/minio/minio-go/v7/pkg/credentials"
16+
)
17+
18+
func TestFileSecretsCredentials(t *testing.T) {
19+
tempDir := t.TempDir()
20+
21+
access, secret := secretsPaths(tempDir)
22+
23+
// Instanciate provider (what we're testing).
24+
provider := &s3.FileSecretsCredentials{
25+
AccessKeyFile: access,
26+
SecretKeyFile: secret,
27+
}
28+
29+
ctx := context.Background()
30+
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
31+
defer cancel()
32+
33+
// Create server
34+
addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
35+
if errors.Is(err, s3testing.ErrMinioNotFound) {
36+
t.Skip("minio binary not found locally, make sure it is in PATH")
37+
}
38+
if err != nil {
39+
t.Fatal(err)
40+
}
41+
defer func() { _ = stop() }()
42+
43+
// Create minio client, using our provider.
44+
creds := credentials.New(provider)
45+
clt, err := minio.New(addr, &minio.Options{
46+
Creds: creds,
47+
Region: "us-east-1",
48+
})
49+
if err != nil {
50+
t.Fatal(err)
51+
}
52+
53+
assertClientSuccess := func(want bool, when string) {
54+
_, err = clt.BucketExists(ctx, "doesnotmatter")
55+
s := "fail"
56+
if want {
57+
s = "succeed"
58+
}
59+
ok := (err == nil) == want
60+
if !ok {
61+
t.Fatalf("expected call to %s %s", s, when)
62+
}
63+
}
64+
65+
// First credential files creation.
66+
// Keep them empty for now,
67+
// so that calls to the server will fail.
68+
writeSecrets(t, tempDir, "")
69+
70+
// The files do not hold the right values,
71+
// so a call to the server should fail.
72+
assertClientSuccess(false, "just after init")
73+
74+
// Write the right keys to the files.
75+
// We're not testing expiry here,
76+
// and forcing credentials cache to update.
77+
writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
78+
creds.Expire()
79+
assertClientSuccess(true, "after changing files content")
80+
81+
// Change content of the files.
82+
writeSecrets(t, tempDir, "badcredentials")
83+
creds.Expire()
84+
assertClientSuccess(false, "after changing again, to bad credentials")
85+
}
86+
87+
func TestBackendWithSecrets(t *testing.T) {
88+
tempDir := t.TempDir()
89+
90+
ctx := context.Background()
91+
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
92+
defer cancel()
93+
94+
addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
95+
if errors.Is(err, s3testing.ErrMinioNotFound) {
96+
t.Skip("minio binary not found locally, make sure it is in PATH")
97+
}
98+
if err != nil {
99+
t.Fatal(err)
100+
}
101+
defer func() { _ = stop() }()
102+
103+
// Prepare backend options to reuse.
104+
// These will not change.
105+
access, secret := secretsPaths(tempDir)
106+
opt := s3.Options{
107+
AccessKeyFile: access,
108+
SecretKeyFile: secret,
109+
Region: "us-east-1",
110+
Bucket: "test-bucket",
111+
CreateBucket: true,
112+
EndpointURL: "http://" + addr,
113+
}
114+
115+
// Backend should not start if secrets files do not exist.
116+
_, err = s3.New(ctx, opt)
117+
if !errors.Is(err, os.ErrNotExist) {
118+
t.Fatal("backend should not start without credentials")
119+
}
120+
121+
// Now write files, but with bad content.
122+
writeSecrets(t, tempDir, "")
123+
_, err = s3.New(ctx, opt)
124+
if err == nil || err.Error() != "Access Denied." {
125+
t.Fatal("backend should not start with bad credentials")
126+
}
127+
128+
// Write the good content.
129+
// Now the backend should start and be able to perform a request.
130+
writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
131+
132+
backend, err := s3.New(ctx, opt)
133+
if err != nil {
134+
t.Fatal(err)
135+
}
136+
_, err = backend.List(ctx, "")
137+
if err != nil {
138+
t.Fatal(err)
139+
}
140+
141+
// Finally, the whole test suite should succeed.
142+
tester.DoBackendTests(t, backend)
143+
}
144+
145+
// secretsPaths returns the file paths for the access key
146+
// and the secret key, respectively.
147+
// For a same dir, the returned values will always be the same.
148+
func secretsPaths(dir string) (access, secret string) {
149+
access = filepath.Join(dir, "access-key")
150+
secret = filepath.Join(dir, "secret-key")
151+
return
152+
}
153+
154+
// writeSecrets writes content to files called "access-key" and "secret-key"
155+
// in dir.
156+
// It returns
157+
func writeSecrets(t testing.TB, dir, content string) {
158+
access, secret := secretsPaths(dir)
159+
err := os.WriteFile(access, []byte(content), 0666)
160+
if err != nil {
161+
t.Fatal(err)
162+
}
163+
err = os.WriteFile(secret, []byte(content), 0666)
164+
if err != nil {
165+
t.Fatal(err)
166+
}
167+
}

‎backends/s3/s3.go‎

Lines changed: 39 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ const (
3737
// DefaultUpdateMarkerForceListInterval is the default value for
3838
// UpdateMarkerForceListInterval.
3939
DefaultUpdateMarkerForceListInterval = 5 * time.Minute
40+
// DefaultSecretsRefreshInterval is the default value for RefreshSecrets.
41+
// It should not be too high so as to retrieve secrets regularly.
42+
DefaultSecretsRefreshInterval = 15 * time.Second
4043
)
4144

4245
// Options describes the storage options for the S3 backend
@@ -45,6 +48,22 @@ type Options struct {
4548
AccessKey string `yaml:"access_key"`
4649
SecretKey string `yaml:"secret_key"`
4750

51+
// Path to the file containing the access key
52+
// as an alternative to AccessKey and SecretKey,
53+
// e.g. /etc/s3-secrets/access-key.
54+
AccessKeyFile string `yaml:"access_key_file"`
55+
56+
// Path to the file containing the secret key
57+
// as an alternative to AccessKey and SecretKey,
58+
// e.g. /etc/s3-secrets/secret-key.
59+
SecretKeyFile string `yaml:"secret_key_file"`
60+
61+
// Time between each secrets retrieval.
62+
// Minimum is 1s, lower values are considered an error.
63+
// It defaults to DefaultSecretsRefreshInterval,
64+
// which is currently 15s.
65+
SecretsRefreshInterval time.Duration `yaml:"secrets_refresh_interval"`
66+
4867
// Region defaults to "us-east-1", which also works for Minio
4968
Region string `yaml:"region"`
5069
Bucket string `yaml:"bucket"`
@@ -93,11 +112,13 @@ type Options struct {
93112
}
94113

95114
func (o Options) Check() error {
96-
if o.AccessKey == "" {
97-
return fmt.Errorf("s3 storage.options: access_key is required")
115+
hasSecretsCreds := o.AccessKeyFile != "" && o.SecretKeyFile != ""
116+
hasStaticCreds := o.AccessKey != "" && o.SecretKey != ""
117+
if !hasSecretsCreds && !hasStaticCreds {
118+
return fmt.Errorf("s3 storage.options: credentials are required, fill either (access_key and secret_key) or (access_key_filename and secret_key_filename)")
98119
}
99-
if o.SecretKey == "" {
100-
return fmt.Errorf("s3 storage.options: secret_key is required")
120+
if hasSecretsCreds && o.SecretsRefreshInterval < time.Second {
121+
return fmt.Errorf("s3 storage.options: field secrets_refresh_interval must be at least 1s")
101122
}
102123
if o.Bucket == "" {
103124
return fmt.Errorf("s3 storage.options: bucket is required")
@@ -289,6 +310,9 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
289310
if opt.EndpointURL == "" {
290311
opt.EndpointURL = DefaultEndpointURL
291312
}
313+
if opt.SecretsRefreshInterval == 0 {
314+
opt.SecretsRefreshInterval = DefaultSecretsRefreshInterval
315+
}
292316
if err := opt.Check(); err != nil {
293317
return nil, err
294318
}
@@ -342,8 +366,17 @@ func New(ctx context.Context, opt Options) (*Backend, error) {
342366
return nil, fmt.Errorf("unsupported scheme for S3: '%s', use http or https.", u.Scheme)
343367
}
344368

369+
creds := credentials.NewStaticV4(opt.AccessKey, opt.SecretKey, "")
370+
if opt.AccessKeyFile != "" {
371+
creds = credentials.New(&FileSecretsCredentials{
372+
AccessKeyFile: opt.AccessKeyFile,
373+
SecretKeyFile: opt.SecretKeyFile,
374+
RefreshInterval: opt.SecretsRefreshInterval,
375+
})
376+
}
377+
345378
cfg := &minio.Options{
346-
Creds: credentials.NewStaticV4(opt.AccessKey, opt.SecretKey, ""),
379+
Creds: creds,
347380
Secure: useSSL,
348381
Transport: hc.Transport,
349382
Region: opt.Region,
@@ -404,7 +437,7 @@ func convertMinioError(err error, isList bool) error {
404437
}
405438
errRes := minio.ToErrorResponse(err)
406439
if !isList && errRes.StatusCode == 404 {
407-
return fmt.Errorf("%w: %s", os.ErrNotExist, err.Error())
440+
return fmt.Errorf("%w: %s", os.ErrNotExist, err.Error())
408441
}
409442
if errRes.Code == "BucketAlreadyOwnedByYou" {
410443
return nil

‎backends/s3/s3testing/minio.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package s3testing
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http"
7+
"os"
8+
"os/exec"
9+
"time"
10+
)
11+
12+
const (
13+
AdminUserOrPassword = "simpleblob"
14+
)
15+
16+
var ErrMinioNotFound = exec.ErrNotFound
17+
18+
// ServeMinio starts a Minio server in the background,
19+
// and waits for it to be ready.
20+
// It returns its address,
21+
// and a function to stop the server gracefully.
22+
//
23+
// The admin username and password for the server are both "simpleblob".
24+
//
25+
// If the minio binary cannot be found in PATH,
26+
// ErrMinioNotFound will be returned.
27+
func ServeMinio(ctx context.Context, dir string) (string, func() error, error) {
28+
cmdname, err := exec.LookPath("minio")
29+
if err != nil {
30+
return "", nil, err
31+
}
32+
33+
port, err := FreePort()
34+
if err != nil {
35+
return "", nil, err
36+
}
37+
addr := fmt.Sprintf("127.0.0.1:%d", port)
38+
39+
cmd := exec.CommandContext(ctx, cmdname, "server", "--quiet", "--address", addr, dir)
40+
cmd.Env = append([]string{}, os.Environ()...)
41+
cmd.Env = append(cmd.Env, "MINIO_BROWSER=off")
42+
cmd.Env = append(cmd.Env, "MINIO_ROOT_USER="+AdminUserOrPassword, "MINIO_ROOT_PASSWORD="+AdminUserOrPassword)
43+
cmd.Stdout = os.Stdout
44+
cmd.Stderr = os.Stderr
45+
if err := cmd.Start(); err != nil {
46+
return "", nil, err
47+
}
48+
49+
// Wait for server to accept requests.
50+
readyURL := "http://" + addr + "/minio/health/ready"
51+
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
52+
defer cancel()
53+
ticker := time.NewTicker(30 * time.Millisecond)
54+
defer ticker.Stop()
55+
for {
56+
select {
57+
case <-ctx.Done():
58+
return "", nil, ctx.Err()
59+
case <-ticker.C:
60+
}
61+
62+
resp, err := http.Get(readyURL)
63+
if err == nil && resp.StatusCode == 200 {
64+
break
65+
}
66+
}
67+
68+
stop := func() error {
69+
_ = cmd.Process.Signal(os.Interrupt)
70+
return cmd.Wait()
71+
}
72+
return addr, stop, nil
73+
}

‎backends/s3/s3testing/port.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package s3testing
2+
3+
import "net"
4+
5+
// FreePort returns a port number free for use.
6+
func FreePort() (int, error) {
7+
l, err := net.Listen("tcp", ":0")
8+
if err != nil {
9+
return 0, err
10+
}
11+
defer l.Close()
12+
addr := l.Addr().(*net.TCPAddr)
13+
return addr.Port, nil
14+
}

0 commit comments

Comments
 (0)