CLOUDP-66615: Add support for rotation of TLS certificates and keys (#122)

Author: fabianlindfors

.evergreen.yml

@@ -151,6 +151,7 @@ task_groups:
     - e2e_test_replica_set_multiple
     - e2e_test_replica_set_tls
     - e2e_test_replica_set_tls_upgrade
+    - e2e_test_replica_set_tls_rotate
   teardown_task:
     - func: upload_e2e_logs
 
@@ -277,6 +278,12 @@ tasks:
         vars:
           test: replica_set_tls_upgrade
 
+  - name: e2e_test_replica_set_tls_rotate
+    commands:
+      - func: run_e2e_test
+        vars:
+          test: replica_set_tls_rotate
+
 buildvariants:
   - name: go_unit_tests
     display_name: go_unit_tests

pkg/apis/mongodb/v1/mongodb_types.go

@@ -197,7 +197,7 @@ func (m MongoDB) TLSSecretNamespacedName() types.NamespacedName {
 // TLSOperatorSecretNamespacedName will get the namespaced name of the Secret created by the operator
 // containing the combined certificate and key.
 func (m MongoDB) TLSOperatorSecretNamespacedName() types.NamespacedName {
-	return types.NamespacedName{Name: "mongodb-operator-server-certificate-key", Namespace: m.Namespace}
+	return types.NamespacedName{Name: m.Name + "-server-certificate-key", Namespace: m.Namespace}
 }
 
 func (m MongoDB) NamespacedName() types.NamespacedName {

pkg/controller/mongodb/mongodb_tls.go

@@ -1,6 +1,7 @@
 package mongodb
 
 import (
+	"crypto/sha256"
 	"fmt"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -23,8 +24,7 @@ const (
 	tlsCAMountPath             = "/var/lib/tls/ca/"
 	tlsCACertName              = "ca.crt"
 	tlsOperatorSecretMountPath = "/var/lib/tls/server/" //nolint
-	tlsOperatorSecretFileName  = "server.pem"
-	tlsSecretCertName          = "tls.crt" //nolint
+	tlsSecretCertName          = "tls.crt"              //nolint
 	tlsSecretKeyName           = "tls.key"
 )
 
@@ -74,6 +74,9 @@ func (r *ReplicaSetReconciler) validateTLSConfig(mdb mdbv1.MongoDB) (bool, error
 		return false, nil
 	}
 
+	// Watch certificate-key secret to handle rotations
+	r.secretWatcher.Watch(mdb.TLSSecretNamespacedName(), mdb.NamespacedName())
+
 	return true, nil
 }
 
@@ -84,47 +87,72 @@ func getTLSConfigModification(getUpdateCreator secret.GetUpdateCreator, mdb mdbv
 		return automationconfig.NOOP(), nil
 	}
 
-	if err := ensureTLSSecret(getUpdateCreator, mdb); err != nil {
+	cert, key, err := getCertAndKey(getUpdateCreator, mdb)
+	if err != nil {
+		return automationconfig.NOOP(), err
+	}
+
+	err = ensureTLSSecret(getUpdateCreator, mdb, cert, key)
+	if err != nil {
 		return automationconfig.NOOP(), err
 	}
 
 	// The config is only updated after the certs and keys have been rolled out to all pods.
 	// The agent needs these to be in place before the config is updated.
 	// Once the config is updated, the agents will gradually enable TLS in accordance with: https://docs.mongodb.com/manual/tutorial/upgrade-cluster-to-ssl/
 	if hasRolledOutTLS(mdb) {
-		return tlsConfigModification(mdb), nil
+		return tlsConfigModification(mdb, cert, key), nil
 	}
 
 	return automationconfig.NOOP(), nil
 }
 
-// ensureTLSSecret will create or update the operator-managed Secret containing
-// the concatenated certificate and key from the user-provided Secret.
-func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB) error {
-	cert, err := secret.ReadKey(getUpdateCreator, tlsSecretCertName, mdb.TLSSecretNamespacedName())
+// getCertAndKey will fetch the certificate and key from the user-provided Secret.
+func getCertAndKey(getter secret.Getter, mdb mdbv1.MongoDB) (string, string, error) {
+	cert, err := secret.ReadKey(getter, tlsSecretCertName, mdb.TLSSecretNamespacedName())
 	if err != nil {
-		return err
+		return "", "", err
 	}
 
-	key, err := secret.ReadKey(getUpdateCreator, tlsSecretKeyName, mdb.TLSSecretNamespacedName())
+	key, err := secret.ReadKey(getter, tlsSecretKeyName, mdb.TLSSecretNamespacedName())
 	if err != nil {
-		return err
+		return "", "", err
 	}
 
+	return cert, key, nil
+}
+
+// ensureTLSSecret will create or update the operator-managed Secret containing
+// the concatenated certificate and key from the user-provided Secret.
+func ensureTLSSecret(getUpdateCreator secret.GetUpdateCreator, mdb mdbv1.MongoDB, cert, key string) error {
+	// Calculate file name from certificate and key
+	fileName := tlsOperatorSecretFileName(cert, key)
+
 	operatorSecret := secret.Builder().
 		SetName(mdb.TLSOperatorSecretNamespacedName().Name).
 		SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace).
-		SetField(tlsOperatorSecretFileName, cert+key).
+		SetField(fileName, cert+key).
 		SetOwnerReferences([]metav1.OwnerReference{getOwnerReference(mdb)}).
 		Build()
 
 	return secret.CreateOrUpdate(getUpdateCreator, operatorSecret)
 }
 
+// tlsOperatorSecretFileName calculates the file name to use for the mounted
+// certificate-key file. The name is based on the hash of the combined cert and key.
+// If the certificate or key changes, the file path changes as well which will trigger
+// the agent to perform a restart.
+// The user-provided secret is being watched and will trigger a reconciliation
+// on changes. This enables the operator to automatically handle cert rotations.
+func tlsOperatorSecretFileName(cert, key string) string {
+	hash := sha256.Sum256([]byte(cert + key))
+	return fmt.Sprintf("%x.pem", hash)
+}
+
 // tlsConfigModification will enable TLS in the automation config.
-func tlsConfigModification(mdb mdbv1.MongoDB) automationconfig.Modification {
+func tlsConfigModification(mdb mdbv1.MongoDB, cert, key string) automationconfig.Modification {
 	caCertificatePath := tlsCAMountPath + tlsCACertName
-	certificateKeyPath := tlsOperatorSecretMountPath + tlsOperatorSecretFileName
+	certificateKeyPath := tlsOperatorSecretMountPath + tlsOperatorSecretFileName(cert, key)
 
 	mode := automationconfig.TLSModeRequired
 	if mdb.Spec.Security.TLS.Optional {

pkg/controller/mongodb/mongodb_tls_test.go

@@ -139,9 +139,11 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 		}, ac.TLS)
 
 		for _, process := range ac.Processes {
+			operatorSecretFileName := tlsOperatorSecretFileName("CERT", "KEY")
+
 			assert.Equal(t, automationconfig.MongoDBTLS{
 				Mode:                               automationconfig.TLSModeRequired,
-				PEMKeyFile:                         tlsOperatorSecretMountPath + tlsOperatorSecretFileName,
+				PEMKeyFile:                         tlsOperatorSecretMountPath + operatorSecretFileName,
 				CAFile:                             tlsCAMountPath + tlsCACertName,
 				AllowConnectionsWithoutCertificate: true,
 			}, process.Args26.Net.TLS)
@@ -160,9 +162,11 @@ func TestAutomationConfig_IsCorrectlyConfiguredWithTLS(t *testing.T) {
 		}, ac.TLS)
 
 		for _, process := range ac.Processes {
+			operatorSecretFileName := tlsOperatorSecretFileName("CERT", "KEY")
+
 			assert.Equal(t, automationconfig.MongoDBTLS{
 				Mode:                               automationconfig.TLSModePreferred,
-				PEMKeyFile:                         tlsOperatorSecretMountPath + tlsOperatorSecretFileName,
+				PEMKeyFile:                         tlsOperatorSecretMountPath + operatorSecretFileName,
 				CAFile:                             tlsCAMountPath + tlsCACertName,
 				AllowConnectionsWithoutCertificate: true,
 			}, process.Args26.Net.TLS)
@@ -182,7 +186,7 @@ func TestTLSOperatorSecret(t *testing.T) {
 
 		// Operator-managed secret should have been created and contain the
 		// concatenated certificate and key.
-		certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName, mdb.TLSOperatorSecretNamespacedName())
+		certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName("CERT", "KEY"), mdb.TLSOperatorSecretNamespacedName())
 		assert.NoError(t, err)
 		assert.Equal(t, "CERTKEY", certificateKey)
 	})
@@ -197,16 +201,17 @@ func TestTLSOperatorSecret(t *testing.T) {
 		s := secret.Builder().
 			SetName(mdb.TLSOperatorSecretNamespacedName().Name).
 			SetNamespace(mdb.TLSOperatorSecretNamespacedName().Namespace).
-			SetField(tlsOperatorSecretFileName, "").
+			SetField(tlsOperatorSecretFileName("", ""), "").
 			Build()
 		err = client.CreateSecret(s)
 		assert.NoError(t, err)
 
 		_, err = getTLSConfigModification(client, mdb)
 		assert.NoError(t, err)
 
-		// Operator-managed secret should have been updated the with concatenated certificate and key.
-		certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName, mdb.TLSOperatorSecretNamespacedName())
+		// Operator-managed secret should have been updated with the concatenated
+		// certificate and key.
+		certificateKey, err := secret.ReadKey(client, tlsOperatorSecretFileName("CERT", "KEY"), mdb.TLSOperatorSecretNamespacedName())
 		assert.NoError(t, err)
 		assert.Equal(t, "CERTKEY", certificateKey)
 	})

pkg/controller/mongodb/replica_set_controller.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/controller/watch"
+
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/persistentvolumeclaim"
 
 	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/probes"
@@ -77,19 +79,22 @@ func Add(mgr manager.Manager) error {
 // contains the list of all available MongoDB versions
 type ManifestProvider func() (automationconfig.VersionManifest, error)
 
-// newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, manifestProvider ManifestProvider) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, manifestProvider ManifestProvider) *ReplicaSetReconciler {
 	mgrClient := mgr.GetClient()
+	secretWatcher := watch.New()
+
 	return &ReplicaSetReconciler{
 		client:           kubernetesClient.NewClient(mgrClient),
 		scheme:           mgr.GetScheme(),
 		manifestProvider: manifestProvider,
 		log:              zap.S(),
+		secretWatcher:    &secretWatcher,
 	}
 }
 
-// add adds a new Controller to mgr with r as the reconcile.Reconciler
-func add(mgr manager.Manager, r reconcile.Reconciler) error {
+// add sets up a controller for the Reconciler on the manager. It will
+// also configure the necessary watches.
+func add(mgr manager.Manager, r *ReplicaSetReconciler) error {
 	// Create a new controller
 	c, err := controller.New("replicaset-controller", mgr, controller.Options{Reconciler: r})
 	if err != nil {
@@ -101,6 +106,12 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error {
 	if err != nil {
 		return err
 	}
+
+	err = c.Watch(&source.Kind{Type: &corev1.Secret{}}, r.secretWatcher)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -115,6 +126,7 @@ type ReplicaSetReconciler struct {
 	scheme           *runtime.Scheme
 	manifestProvider func() (automationconfig.VersionManifest, error)
 	log              *zap.SugaredLogger
+	secretWatcher    *watch.ResourceWatcher
 }
 
 // Reconcile reads that state of the cluster for a MongoDB object and makes changes based on the state read

test/e2e/replica_set_tls_rotate/replica_set_tls_rotate_test.go

@@ -0,0 +1,49 @@
+package replica_set_tls
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/tlstests"
+
+	e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests"
+	setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup"
+	f "github.com/operator-framework/operator-sdk/pkg/test"
+)
+
+func TestMain(m *testing.M) {
+	f.MainEntry(m)
+}
+
+func TestReplicaSetTLSRotate(t *testing.T) {
+	ctx, shouldCleanup := setup.InitTest(t)
+	if shouldCleanup {
+		defer ctx.Cleanup()
+	}
+
+	mdb, user := e2eutil.NewTestMongoDB("mdb-tls")
+	mdb.Spec.Security.TLS = e2eutil.NewTestTLSConfig(false)
+
+	_, err := setup.GeneratePasswordForUser(user, ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if err := setup.CreateTLSResources(mdb.Namespace, ctx); err != nil {
+		t.Fatalf("Failed to set up TLS resources: %+v", err)
+	}
+
+	t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
+	t.Run("Basic tests", mongodbtests.BasicFunctionality(&mdb))
+	t.Run("Wait for TLS to be enabled", tlstests.WaitForTLSMode(&mdb, "requireSSL"))
+	t.Run("Test Basic TLS Connectivity", tlstests.ConnectivityWithTLS(&mdb))
+	t.Run("Test TLS required", tlstests.ConnectivityWithoutTLSShouldFail(&mdb))
+
+	t.Run("MongoDB is reachable while certificate is rotated", tlstests.IsReachableOverTLSDuring(&mdb, time.Second*10,
+		func() {
+			t.Run("Update certificate secret", tlstests.RotateCertificate(&mdb))
+			t.Run("Wait for certificate to be rotated", tlstests.WaitForRotatedCertificate(&mdb))
+		},
+	))
+}

test/e2e/tlstests/tlstests.go

@@ -6,9 +6,14 @@ import (
 	"crypto/x509"
 	"fmt"
 	"io/ioutil"
+	"math/big"
 	"testing"
 	"time"
 
+	f "github.com/operator-framework/operator-sdk/pkg/test"
+
+	"github.com/mongodb/mongodb-kubernetes-operator/pkg/kube/secret"
+
 	v1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
 	e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
 	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests"
@@ -136,6 +141,59 @@ func getAdminSetting(uri, key string) (interface{}, error) {
 	return value, nil
 }
 
+func RotateCertificate(mdb *v1.MongoDB) func(*testing.T) {
+	return func(t *testing.T) {
+		// Load new certificate and key
+		cert, err := ioutil.ReadFile("testdata/tls/server_rotated.crt")
+		assert.NoError(t, err)
+		key, err := ioutil.ReadFile("testdata/tls/server_rotated.key")
+		assert.NoError(t, err)
+
+		certKeySecret := secret.Builder().
+			SetName(mdb.Spec.Security.TLS.CertificateKeySecret.Name).
+			SetNamespace(mdb.Namespace).
+			SetField("tls.crt", string(cert)).
+			SetField("tls.key", string(key)).
+			Build()
+
+		err = f.Global.Client.Update(context.TODO(), &certKeySecret)
+		assert.NoError(t, err)
+	}
+}
+
+func WaitForRotatedCertificate(mdb *v1.MongoDB) func(*testing.T) {
+	return func(t *testing.T) {
+		// The rotated certificate has serial number 2
+		expectedSerial := big.NewInt(2)
+
+		tlsConfig, err := getClientTLSConfig()
+		assert.NoError(t, err)
+
+		// Reject all server certificates that don't have the expected serial number
+		tlsConfig.VerifyPeerCertificate = func(rawCerts [][]byte, verifiedChains [][]*x509.Certificate) error {
+			cert := verifiedChains[0][0]
+			if expectedSerial.Cmp(cert.SerialNumber) != 0 {
+				return fmt.Errorf("expected certificate serial number %s, got %s", expectedSerial, cert.SerialNumber)
+			}
+
+			return nil
+		}
+
+		opts := options.Client().SetTLSConfig(tlsConfig).ApplyURI(mdb.MongoURI())
+		mongoClient, err := mongo.Connect(context.TODO(), opts)
+		assert.NoError(t, err)
+
+		// Ping the cluster until it succeeds. The ping will only suceed with the right certificate.
+		err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
+			if err := mongoClient.Ping(context.TODO(), nil); err != nil {
+				return false, nil
+			}
+			return true, nil
+		})
+		assert.NoError(t, err)
+	}
+}
+
 func getClientTLSConfig() (*tls.Config, error) {
 	// Read the CA certificate from test data
 	caPEM, err := ioutil.ReadFile("testdata/tls/ca.crt")