feat: [FFM-12184]: add ability to rotate auth secrets (#385)

* feat: [FFM-12184]: allow legacy auth keys to decode tokens

* feat: [FFM-12184]: add prometheus metrics for auth secret vs legacy secret decode metrics

Author: conormurray95

Makefile

@@ -161,7 +161,7 @@ $(GOBIN)/golangci-lint:
 # Install goimports to format code
 $(GOBIN)/goimports:
 	@echo "🔘 Installing goimports ... (`date '+%H:%M:%S'`)"
-	@go install golang.org/x/tools/cmd/goimports@latest
+	@go install golang.org/x/tools/cmd/goimports@v0.30.0
 
 # Install gocov to parse code coverage
 $(GOBIN)/gocov:

cmd/ff-proxy/main.go

@@ -52,6 +52,7 @@ var (
 	clientService         string
 	metricService         string
 	authSecret            string
+	legacyAuthSecrets     legacySecrets
 	metricPostDuration    int
 	heartbeatInterval     int
 	generateOfflineConfig bool
@@ -88,13 +89,30 @@ var (
 	andRules bool
 )
 
+// legacySecrets implements the flag.Value interface and allows us to pass a comma separated
+// list of legacy secrets e.g. -legacy-secrets mysecret,mysecret2
+type legacySecrets []string
+
+func (l *legacySecrets) String() string {
+	return strings.Join(*l, ",")
+}
+
+func (l *legacySecrets) Set(value string) error {
+	ss := strings.Split(value, ",")
+	for _, s := range ss {
+		*l = append(*l, s)
+	}
+	return nil
+}
+
 // Environment Variables
 const (
 	// Service Config
 	proxyKeyEnv              = "PROXY_KEY"
 	clientServiceEnv         = "CLIENT_SERVICE"
 	metricServiceEnv         = "METRIC_SERVICE"
 	authSecretEnv            = "AUTH_SECRET"
+	legacySecretsEnv         = "LEGACY_SECRETS"
 	metricPostDurationEnv    = "METRIC_POST_DURATION"
 	heartbeatIntervalEnv     = "HEARTBEAT_INTERVAL"
 	generateOfflineConfigEnv = "GENERATE_OFFLINE_CONFIG"
@@ -138,6 +156,7 @@ const (
 	clientServiceFlag         = "client-service"
 	metricServiceFlag         = "metric-service"
 	authSecretFlag            = "auth-secret"
+	legacySecretsFlag         = "legacy-secrets"
 	metricPostDurationFlag    = "metric-post-duration"
 	heartbeatIntervalFlag     = "heartbeat-interval"
 	generateOfflineConfigFlag = "generate-offline-config"
@@ -181,6 +200,7 @@ func init() {
 	flag.StringVar(&clientService, clientServiceFlag, "https://config.ff.harness.io/api/1.0", "the url of the ff client service")
 	flag.StringVar(&metricService, metricServiceFlag, "https://events.ff.harness.io/api/1.0", "the url of the ff metric service")
 	flag.StringVar(&authSecret, authSecretFlag, "secret", "the secret used for signing auth tokens")
+	flag.Var(&legacyAuthSecrets, legacySecretsFlag, "legacy secrets used to decode auth tokens. If rotating a secret you can place the old auth-secret in here so old tokens will still remain valid while new tokens are issued using the new auth secret")
 	flag.IntVar(&metricPostDuration, metricPostDurationFlag, 60, "How often in seconds the proxy posts metrics to Harness. Set to 0 to disable.")
 	flag.IntVar(&heartbeatInterval, heartbeatIntervalFlag, 60, "How often in seconds the proxy polls pings it's health function. Set to 0 to disable.")
 	flag.BoolVar(&generateOfflineConfig, generateOfflineConfigFlag, false, "if true the proxy will produce offline config in the /config directory then terminate")
@@ -223,6 +243,7 @@ func init() {
 		clientServiceEnv:                clientServiceFlag,
 		metricServiceEnv:                metricServiceFlag,
 		authSecretEnv:                   authSecretFlag,
+		legacySecretsEnv:                legacySecretsFlag,
 		redisAddrEnv:                    redisAddressFlag,
 		redisPasswordEnv:                redisPasswordFlag,
 		redisUsernameEnv:                redisUsernameFlag,
@@ -561,6 +582,10 @@ func main() {
 	})
 
 	// Configure endpoints and server
+	legacySecretsBytes := make([][]byte, 0)
+	for _, s := range legacyAuthSecrets {
+		legacySecretsBytes = append(legacySecretsBytes, []byte(s))
+	}
 	endpoints := transport.NewEndpoints(service)
 	server := transport.NewHTTPServer(port, endpoints, logger, tlsEnabled, tlsCert, tlsKey)
 	server.Use(
@@ -569,7 +594,7 @@ func main() {
 		middleware.NewCorsMiddleware(),
 		middleware.NewEchoRequestIDMiddleware(),
 		middleware.NewEchoLoggingMiddleware(logger),
-		middleware.NewEchoAuthMiddleware(logger, authRepo, []byte(authSecret), bypassAuth),
+		middleware.NewEchoAuthMiddleware(logger, authRepo, []byte(authSecret), legacySecretsBytes, bypassAuth, promReg),
 		middleware.ValidateEnvironment(bypassAuth),
 	)
 

middleware/middleware.go

@@ -11,12 +11,11 @@ import (
 
 	"github.com/golang-jwt/jwt"
 	"github.com/google/uuid"
+	"github.com/harness/ff-proxy/v2/domain"
+	"github.com/harness/ff-proxy/v2/log"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
 	"github.com/prometheus/client_golang/prometheus"
-
-	"github.com/harness/ff-proxy/v2/domain"
-	"github.com/harness/ff-proxy/v2/log"
 )
 
 type requestContextKey string
@@ -52,29 +51,51 @@ func NewEchoLoggingMiddleware(l log.Logger) echo.MiddlewareFunc {
 	})
 }
 
+// validateToken attempts to validate a JWT token with the given secret
+func validateToken(tokenStr string, secret []byte) (*jwt.Token, error) {
+	return jwt.ParseWithClaims(tokenStr, &domain.Claims{}, func(t *jwt.Token) (interface{}, error) {
+		return secret, nil
+	})
+}
+
 // NewEchoAuthMiddleware returns an echo middleware that checks if auth headers
 // are valid
-func NewEchoAuthMiddleware(logger log.Logger, authRepo keyLookUp, secret []byte, bypassAuth bool) echo.MiddlewareFunc {
+// nolint:cyclop
+func NewEchoAuthMiddleware(logger log.Logger, authRepo keyLookUp, secret []byte, legacySecrets [][]byte, bypassAuth bool, reg *prometheus.Registry) echo.MiddlewareFunc {
+	metrics := newPrometheusAuth(reg)
+
 	return middleware.JWTWithConfig(middleware.JWTConfig{
 		AuthScheme:  "Bearer",
 		TokenLookup: "header:Authorization",
 		ParseTokenFunc: func(auth string, c echo.Context) (interface{}, error) {
 			if auth == "" {
-				return nil, errors.New("token was empty")
+				return nil, errors.New("authorization token is required")
 			}
 
-			token, err := jwt.ParseWithClaims(auth, &domain.Claims{}, func(t *jwt.Token) (interface{}, error) {
-				return secret, nil
-			})
-			if err != nil {
-				return nil, err
+			// First try with current secret
+			token, err := validateToken(auth, secret)
+			if err == nil {
+				metrics.currentSecretTokens.Inc()
+				if claims, ok := token.Claims.(*domain.Claims); ok && token.Valid && isKeyInCache(c.Request().Context(), logger, authRepo, claims) {
+					c.Set(tokenClaims.String(), claims)
+					return nil, nil
+				}
+				return nil, errors.New("invalid token")
 			}
 
-			if claims, ok := token.Claims.(*domain.Claims); ok && token.Valid && isKeyInCache(c.Request().Context(), logger, authRepo, claims) {
-				c.Set(tokenClaims.String(), claims)
-				return nil, nil
+			// If current secret fails, try legacy secrets
+			for _, legacySecret := range legacySecrets {
+				if token, err = validateToken(auth, legacySecret); err == nil {
+					metrics.legacySecretTokens.Inc()
+					if claims, ok := token.Claims.(*domain.Claims); ok && token.Valid && isKeyInCache(c.Request().Context(), logger, authRepo, claims) {
+						c.Set(tokenClaims.String(), claims)
+						return nil, nil
+					}
+					return nil, errors.New("invalid token")
+				}
 			}
-			return nil, errors.New("invalid token")
+
+			return nil, err
 		},
 		Skipper: func(c echo.Context) bool {
 			if bypassAuth {

middleware/middleware_test.go

@@ -1,15 +1,21 @@
 package middleware
 
 import (
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strings"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/golang-jwt/jwt"
+
 	"github.com/harness/ff-proxy/v2/domain"
 	"github.com/labstack/echo/v4"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAllowQuerySemicolons(t *testing.T) {
@@ -216,3 +222,111 @@ func TestSkipper(t *testing.T) {
 		})
 	}
 }
+
+type mockKeyLookup struct {
+	shouldExist bool
+}
+
+func (m *mockKeyLookup) Get(_ context.Context, _ domain.AuthAPIKey) (string, bool, error) {
+	return "", m.shouldExist, nil
+}
+
+func TestNewEchoAuthMiddleware(t *testing.T) {
+	// Create test secrets
+	currentSecret := []byte("current-secret")
+	legacySecret := []byte("legacy-secret")
+	legacySecrets := [][]byte{legacySecret}
+
+	// Create test claims
+	validClaims := &domain.Claims{
+		APIKey: "test-key",
+	}
+
+	// Generate test tokens
+	validToken, err := jwt.NewWithClaims(jwt.SigningMethodHS256, validClaims).SignedString(currentSecret)
+	require.NoError(t, err)
+	legacyToken, err := jwt.NewWithClaims(jwt.SigningMethodHS256, validClaims).SignedString(legacySecret)
+	require.NoError(t, err)
+	invalidToken := "invalid-token"
+
+	tests := []struct {
+		name       string
+		token      string
+		keyExists  bool
+		bypassAuth bool
+		wantStatus int
+	}{
+		{
+			name:       "Valid token with current secret",
+			token:      validToken,
+			keyExists:  true,
+			wantStatus: http.StatusOK,
+		},
+		{
+			name:       "Valid token with legacy secret",
+			token:      legacyToken,
+			keyExists:  true,
+			wantStatus: http.StatusOK,
+		},
+		{
+			name:       "Invalid token",
+			token:      invalidToken,
+			keyExists:  true,
+			wantStatus: http.StatusUnauthorized,
+		},
+		{
+			name:       "Empty token",
+			token:      "",
+			keyExists:  true,
+			wantStatus: http.StatusUnauthorized,
+		},
+		{
+			name:       "Valid token but key not in cache",
+			token:      validToken,
+			keyExists:  false,
+			wantStatus: http.StatusUnauthorized,
+		},
+		{
+			name:       "Bypass auth enabled",
+			token:      "",
+			bypassAuth: true,
+			wantStatus: http.StatusOK,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// Setup
+			e := echo.New()
+			req := httptest.NewRequest(http.MethodGet, "/", nil)
+			if tt.token != "" {
+				req.Header.Set("Authorization", "Bearer "+tt.token)
+			}
+			rec := httptest.NewRecorder()
+			c := e.NewContext(req, rec)
+
+			// Create mock auth repo with desired behavior
+			mockRepo := &mockKeyLookup{shouldExist: tt.keyExists}
+
+			// Create middleware
+			middleware := NewEchoAuthMiddleware(nil, mockRepo, currentSecret, legacySecrets, tt.bypassAuth, prometheus.NewRegistry())
+
+			// Create test handler
+			handler := middleware(func(c echo.Context) error {
+				return c.NoContent(http.StatusOK)
+			})
+
+			// Execute
+			_ = handler(c)
+
+			// Assert response status code
+			assert.Equal(t, tt.wantStatus, rec.Code)
+
+			// For successful non-bypassed auth, verify claims are set
+			if tt.wantStatus == http.StatusOK && !tt.bypassAuth {
+				claims := c.Get(tokenClaims.String()).(*domain.Claims)
+				assert.Equal(t, validClaims.APIKey, claims.APIKey)
+			}
+		})
+	}
+}

middleware/prometheus.go

@@ -0,0 +1,27 @@
+package middleware
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// prometheusAuth is used for tracking prometheus metrics around auth token validation
+type prometheusAuth struct {
+	currentSecretTokens prometheus.Counter
+	legacySecretTokens  prometheus.Counter
+}
+
+func newPrometheusAuth(reg *prometheus.Registry) *prometheusAuth {
+	p := &prometheusAuth{
+		currentSecretTokens: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "ff_proxy_auth_current_secret_tokens_total",
+			Help: "The total number of auth tokens decoded with the current secret",
+		}),
+		legacySecretTokens: prometheus.NewCounter(prometheus.CounterOpts{
+			Name: "ff_proxy_auth_legacy_secret_tokens_total",
+			Help: "The total number of auth tokens decoded with a legacy secret",
+		}),
+	}
+
+	reg.MustRegister(p.currentSecretTokens, p.legacySecretTokens)
+	return p
+}

transport/http_server_test.go

@@ -308,7 +308,7 @@ func setupHTTPServer(t *testing.T, bypassAuth bool, opts ...setupOpts) *HTTPServ
 		middleware.AllowQuerySemicolons(),
 		middleware.NewEchoRequestIDMiddleware(),
 		middleware.NewEchoLoggingMiddleware(logger),
-		middleware.NewEchoAuthMiddleware(logger, repo, []byte(`secret`), bypassAuth),
+		middleware.NewEchoAuthMiddleware(logger, repo, []byte(`secret`), [][]byte{}, bypassAuth, prometheus.NewRegistry()),
 		middleware.ValidateEnvironment(bypassAuth),
 	)
 	return server