THREESCALE-11156 add tls env var to zync and system deployments

THREESCALE-11156 add volume mounts for secrets

THREESCALE-11156 fix default internal to work without tls

THREESCALE-11156 set cert permissions

THREESCALE-11156 add apimanger flags for TLS

THREESCALE-11156 fix e2e

THREESCALE-11156 add Database TLS doc

THREESCALE-11156 add watchby logic

THREESCALE-11156 removed internal database changes

apis/apps/v1alpha1/apimanager_types.go

@@ -84,6 +84,10 @@ type APIManagerSpec struct {
 	PodDisruptionBudget *PodDisruptionBudgetSpec `json:"podDisruptionBudget,omitempty"`
 	// +optional
 	Monitoring *MonitoringSpec `json:"monitoring,omitempty"`
+	// +optional
+	SystemDatabaseTLSEnabled *bool `json:"systemDatabaseTLSEnabled,omitempty"`
+	// +optional
+	ZyncDatabaseTLSEnabled *bool `json:"zyncDatabaseTLSEnabled,omitempty"`
 }
 
 // APIManagerStatus defines the observed state of APIManager
@@ -1438,6 +1442,24 @@ func (a *APIManager) GetApicastCustomPoliciesSecretRefs() []*v1.LocalObjectRefer
 	return secretRefs
 }
 
+func (a *APIManager) GetSystemDatabaseSecretRefs() []*v1.LocalObjectReference {
+	secretRefs := []*v1.LocalObjectReference{}
+	systemDatabaseSecretRef := v1.LocalObjectReference{
+		Name: "system-database",
+	}
+	secretRefs = append(secretRefs, &systemDatabaseSecretRef)
+	return secretRefs
+}
+
+func (a *APIManager) GetZyncSecretRefs() []*v1.LocalObjectReference {
+	secretRefs := []*v1.LocalObjectReference{}
+	zyncSecretRef := v1.LocalObjectReference{
+		Name: "zync",
+	}
+	secretRefs = append(secretRefs, &zyncSecretRef)
+	return secretRefs
+}
+
 func (apimanager *APIManager) Get3scaleSecretRefs() []*v1.LocalObjectReference {
 	secretRefs := []*v1.LocalObjectReference{}
 
@@ -1463,6 +1485,16 @@ func (apimanager *APIManager) Get3scaleSecretRefs() []*v1.LocalObjectReference {
 		secretRefs = append(secretRefs, apicastCustomPoliciesSecretRefs...)
 	}
 
+	systemDatabaseSecretRefs := apimanager.GetSystemDatabaseSecretRefs()
+	if len(systemDatabaseSecretRefs) > 0 {
+		secretRefs = append(secretRefs, systemDatabaseSecretRefs...)
+	}
+
+	zyncSecretRefs := apimanager.GetZyncSecretRefs()
+	if len(zyncSecretRefs) > 0 {
+		secretRefs = append(secretRefs, zyncSecretRefs...)
+	}
+
 	secretRefs = removeDuplicateSecretRefs(secretRefs)
 
 	return secretRefs
@@ -1672,3 +1704,11 @@ type APIManagerList struct {
 func init() {
 	SchemeBuilder.Register(&APIManager{}, &APIManagerList{})
 }
+
+func (apimanager *APIManager) IsSystemDatabaseTLSEnabled() bool {
+	return apimanager.Spec.SystemDatabaseTLSEnabled != nil && *apimanager.Spec.SystemDatabaseTLSEnabled
+}
+
+func (apimanager *APIManager) IsZyncDatabaseTLSEnabled() bool {
+	return apimanager.Spec.ZyncDatabaseTLSEnabled != nil && *apimanager.Spec.ZyncDatabaseTLSEnabled
+}

apis/apps/v1alpha1/apimanager_types_test.go

@@ -263,7 +263,14 @@ func TestAPIManager_Get3scaleSecretRefs(t *testing.T) {
 					},
 				},
 			},
-			want: []*v1.LocalObjectReference{},
+			want: []*v1.LocalObjectReference{
+				{
+					Name: "system-database",
+				},
+				{
+					Name: "zync",
+				},
+			},
 		},
 		{
 			name: "Apicast has secret refs",
@@ -319,6 +326,12 @@ func TestAPIManager_Get3scaleSecretRefs(t *testing.T) {
 				{
 					Name: "custom-policy-1-secret",
 				},
+				{
+					Name: "system-database",
+				},
+				{
+					Name: "zync",
+				},
 			},
 		},
 	}

bundle/manifests/apps.3scale.net_apimanagers.yaml

@@ -16603,6 +16603,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              systemDatabaseTLSEnabled:
+                type: boolean
               tenantName:
                 type: string
               wildcardDomain:
@@ -20010,6 +20012,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              zyncDatabaseTLSEnabled:
+                type: boolean
             required:
             - wildcardDomain
             type: object

config/crd/bases/apps.3scale.net_apimanagers.yaml

@@ -17393,6 +17393,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              systemDatabaseTLSEnabled:
+                type: boolean
               tenantName:
                 type: string
               wildcardDomain:
@@ -20960,6 +20962,8 @@ spec:
                         type: array
                     type: object
                 type: object
+              zyncDatabaseTLSEnabled:
+                type: boolean
             required:
             - wildcardDomain
             type: object

doc/apimanager-reference.md

@@ -739,6 +739,11 @@ For Mysql:
 | URL | URL of the Porta database. Format: `mysql2://<AdminUser>:<AdminPassword>@<DatabaseHost>/<DatabaseName>`, where `<AdminUser>` must be an already existing user in the external database with full permissions on the specified `<DatabaseName>` logical database and `<DatabaseName>` must be an already existing logical database in the external database.| Mandatory when the instance is managed externally. A default is only set when database is managed internally.<br/>When managed internally:<br/>`mysql2://root:<AutogeneratedValue>@system-mysql/mysql`.|
 | DB_USER | Not used by 3scale components. Only used when the database is managed internally to create a new user granted with superuser permissions for the database specified in the `URL` field. | `mysql` |
 | DB_PASSWORD | Not used by 3scale components. Only used when the database is managed internally to create a new user granted with superuser permissions for the database specified in the `URL` field. | Autogenerated value |
+| DATABASE_SSL_MODE | [Database SSL Mode](https://github.com/brianmario/mysql2?tab=readme-ov-file#ssltls-options) | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CA | SSL CA certificate | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CERT | SSL CERT certificate | Required to set TLS Database connection. Only for TLS |
+DB_SSL_KEY | SSL Key  | Required to set TLS Database connection. Only for TLS |
+
 
 For Postgresql:
 
@@ -747,6 +752,11 @@ For Postgresql:
 | URL | URL of the Porta database. Format: `postgresql://<AdminUser>:<AdminPassword>@<DatabaseHost>/<DatabaseName>`, where `<AdminUser>` must be an already existing user in the external database with full permissions on the specified `<DatabaseName>` logical database and `<DatabaseName>` must be an already existing logical database in the external database.| Mandatory when the instance is managed externally. A default is only set when database is managed internally.<br/>When managed internally:<br/>`postgresql://system:<AutoGeneratedValue>@system-postgresql/system`.|
 | DB_USER | Not used by 3scale components. Only used when the database is managed internally to create a user with superuser power. | `system` |
 | DB_PASSWORD | Not used by 3scale components. Only used when the database is managed internally to create a user with superuser power. | Autogenerated value |
+| DATABASE_SSL_MODE | [Database SSL Mode](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CA | SSL CA certificate | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CERT | SSL CERT certificate | Required to set TLS Database connection. Only for TLS |
+DB_SSL_KEY | SSL Key  | Required to set TLS Database connection. Only for TLS |
+
 
 For Oracle:
 
@@ -813,6 +823,11 @@ For Oracle:
 | ZYNC_DATABASE_PASSWORD | Database password associated to the user specified in the `DATABASE_URL` parameter | When the database is managed externally, this parameter is mandatory and must have the same value as the password part of the `DATABASE_URL` parameter in this secret. Otherwise the default value is an autogenerated value if not defined |
 | SECRET_KEY_BASE | Zync's application key generator to encrypt communications | Autogenerated value |
 | ZYNC_AUTHENTICATION_TOKEN | Authentication token used to authenticate System when calling Zync | Autogenerated value |
+| DATABASE_SSL_MODE | [Database SSL Mode](https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-PROTECTION) | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CA | SSL CA certificate | Required to set TLS Database connection. Only for TLS |
+| DB_SSL_CERT | SSL CERT certificate | Required to set TLS Database connection. Only for TLS |
+DB_SSL_KEY | SSL Key  | Required to set TLS Database connection. Only for TLS |
+
 
 ### fileStorage-S3-credentials-secret
 

doc/operator-user-guide.md

@@ -278,6 +278,48 @@ Secret name must be `zync`.
 
 See [Zync secret](apimanager-reference.md#zync) for reference.
 
+#### TLS database configuration ####
+
+It is possible to connect to both the system-database and zync database via TLS provided these databases have TLS enabled. To enable TLS communication to these databases you will need to configure the ApiManager and the database secret.
+
+In ApiManager CR we set the boolean to enable TLS configuration for the respictive databases
+- `spec.zyncDatabaseTLSEnabled: true`
+- `spec.systemDatabaseTLSEnabled: true`
+
+We pass the cert files in via the respective secret i.e. system-database & zync
+
+You set the following values in the secret to connect to the database via TLS
+
+| Secret Key | Secret Value |
+| --- | --- |
+| DATABASE_SSL_MODE | string of the SSL mode for database connection |
+| DB_SSL_CA | actual ca cert |
+| DB_SSL_CERT | actual client cert |
+| DB_SSL_KEY | actual client key |
+
+e.g. for system-database
+```bash
+oc create secret generic system-database \
+  --from-literal=DATABASE_SSL_MODE=verify-ca \
+  --from-literal=DATABASE_URL=postgresql://postgres:[email protected]/zync_production \
+  --from-literal=ZYNC_DATABASE_PASSWORD=password \
+  --from-file=DB_SSL_CA=rootCA.crt \
+  --from-file=DB_SSL_CERT=client.crt \
+  --from-file=DB_SSL_KEY=client.key 
+```
+e.g. for zync
+```bash
+oc create secret generic zync \
+  --from-literal=DATABASE_SSL_MODE=verify-ca \
+  --from-literal=DATABASE_URL=postgresql://postgres:[email protected]/zync_production \
+  --from-literal=ZYNC_DATABASE_PASSWORD=password \
+  --from-file=DB_SSL_CA=rootCA.crt \
+  --from-file=DB_SSL_CERT=client.crt \
+  --from-file=DB_SSL_KEY=client.key 
+```
+
+Once these values have been set and are correct the operator will proceed to mount the certs into the related pods to enable client TLS communication.
+
 #### S3 Filestorage Installation
 3scale’s FileStorage being in a S3 service instead of in a PVC.
 

pkg/3scale/amp/component/deployment_annotations.go

@@ -17,7 +17,7 @@ import (
 
 func ComputeWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, deploymentName, watchNS string, component interface{}) (map[string]string, error) {
 	// First get the initial annotations
-	uncheckedAnnotations, err := getWatchedSecretAnnotations(ctx, client, deploymentName, component)
+	uncheckedAnnotations, err := getWatchedSecretAnnotations(ctx, client, deploymentName, watchNS, component)
 	if err != nil {
 		return nil, err
 	}
@@ -74,7 +74,7 @@ func ComputeWatchedSecretAnnotations(ctx context.Context, client k8sclient.Clien
 	return uncheckedAnnotations, nil // No difference with existing annotations so can return uncheckedAnnotations
 }
 
-func getWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, deploymentName string, component interface{}) (map[string]string, error) {
+func getWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, deploymentName string, namespace string, component interface{}) (map[string]string, error) {
 	annotations := map[string]string{}
 
 	switch c := component.(type) {
@@ -192,6 +192,53 @@ func getWatchedSecretAnnotations(ctx context.Context, client k8sclient.Client, d
 				}
 			}
 		}
+	case *System:
+		system := c
+		systemDatabase := &corev1.Secret{}
+		systemDatabaseSecretKey := k8sclient.ObjectKey{
+			Name:      SystemSecretSystemDatabaseSecretName,
+			Namespace: system.Options.Namespace,
+		}
+		err := client.Get(ctx, systemDatabaseSecretKey, systemDatabase)
+		if err != nil {
+			return nil, err
+		}
+		if helper.IsSecretWatchedBy3scale(systemDatabase) {
+			annotationKey := fmt.Sprintf("%s%s", SystemDatabaseSecretResverAnnotationPrefix, systemDatabase.Name)
+			annotations[annotationKey] = systemDatabase.ResourceVersion
+		}
+
+	case *SystemSearchd:
+		systemDatabase := &corev1.Secret{}
+		systemDatabaseSecretKey := k8sclient.ObjectKey{
+			Name:      SystemSecretSystemDatabaseSecretName,
+			Namespace: namespace,
+		}
+		err := client.Get(ctx, systemDatabaseSecretKey, systemDatabase)
+		if err != nil {
+			return nil, err
+		}
+		if helper.IsSecretWatchedBy3scale(systemDatabase) {
+			annotationKey := fmt.Sprintf("%s%s", SystemDatabaseSecretResverAnnotationPrefix, systemDatabase.Name)
+			annotations[annotationKey] = systemDatabase.ResourceVersion
+		}
+
+	case *Zync:
+		zync := c
+		zyncSecret := &corev1.Secret{}
+		zyncSecretKey := k8sclient.ObjectKey{
+			Name:      ZyncSecretName,
+			Namespace: zync.Options.Namespace,
+		}
+		err := client.Get(ctx, zyncSecretKey, zyncSecret)
+		if err != nil {
+			fmt.Printf("failed to find zync secret, yet to be create %s", err)
+			return nil, nil
+		}
+		if helper.IsSecretWatchedBy3scale(zyncSecret) {
+			annotationKey := fmt.Sprintf("%s%s", ZyncSecretResverAnnotationPrefix, zyncSecret.Name)
+			annotations[annotationKey] = zyncSecret.ResourceVersion
+		}
 
 	default:
 		return nil, fmt.Errorf("unrecognized component %s is not supported", deploymentName)
@@ -223,6 +270,27 @@ func HasSecretHashChanged(ctx context.Context, client k8sclient.Client, deployme
 		default:
 			return false
 		}
+	case *System:
+		switch {
+		case strings.HasPrefix(deploymentAnnotation, SystemDatabaseSecretResverAnnotationPrefix):
+			secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, SystemDatabaseSecretResverAnnotationPrefix)
+		default:
+			return false
+		}
+	case *SystemSearchd:
+		switch {
+		case strings.HasPrefix(deploymentAnnotation, SystemDatabaseSecretResverAnnotationPrefix):
+			secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, SystemDatabaseSecretResverAnnotationPrefix)
+		default:
+			return false
+		}
+	case *Zync:
+		switch {
+		//case strings.HasPrefix(deploymentAnnotation, ZyncSecretResverAnnotationPrefix):
+		//	secretToCheckKey.Name = strings.TrimPrefix(deploymentAnnotation, ZyncSecretResverAnnotationPrefix)
+		default:
+			return false
+		}
 	default:
 		logger.Info(fmt.Sprintf("unrecognized component %s is not supported", c))
 		return false