Merge pull request #10 from osodevops/confluent_upgrade

Confluent upgrade

README.md

@@ -150,7 +150,7 @@ File a GitHub [issue](https://github.com/osodevops/kafka-gitops-examples/issues)
 
 ## The legals
 
-Copyright © 2017-2021 [OSO](https://oso.sh) | See [LICENCE](LICENSE) for full details.
+Copyright © 2017-2022 [OSO](https://oso.sh) | See [LICENCE](LICENSE) for full details.
 
 [<img src="https://oso-public-resources.s3.eu-west-1.amazonaws.com/oso-logo-green.png" alt="OSO who we are" width="250"/>](https://oso.sh/who-we-are/)
 

flux-system/gotk-sync.yaml

@@ -26,7 +26,7 @@ spec:
     - apiVersion: helm.toolkit.fluxcd.io/v1beta1
       kind: HelmRelease
       name: confluent
-      namespace: sandbox
+      namespace: confluent
   validation: client
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta1
@@ -38,7 +38,7 @@ spec:
   dependsOn:
     - name: confluent-infra
   interval: 5m
-  path: ./kustomize/environments/sandbox
+  path: ./kustomize/environments
   prune: true
   sourceRef:
     kind: GitRepository

kustomize/environments/sandbox/kafka-connect.yaml → kustomize/base/confluent/connect/kafka-connect.yaml

@@ -1,12 +1,18 @@
+---
 apiVersion: platform.confluent.io/v1beta1
 kind: Connect
 metadata:
   name: connect
 spec:
+  replicas: 1
+  image:
+    application: confluentinc/cp-server-connect:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
   tls:
     autoGeneratedCerts: true
   dependencies:
     kafka:
-      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      bootstrapEndpoint: kafka:9071
       tls:
         enabled: true
+

kustomize/base/confluent/connect/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - kafka-connect.yaml
+

kustomize/base/confluent/control-centre.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: platform.confluent.io/v1beta1
 kind: ControlCenter
 metadata:
@@ -11,6 +12,27 @@ spec:
         failureThreshold: 5
         timeoutSeconds: 500
   image:
-    application: confluentinc/cp-enterprise-control-center-operator:6.1.1.0
-    init: confluentinc/cp-init-container-operator:6.1.1.0
-  dataVolumeCapacity: 10Gi
+    application: confluentinc/cp-enterprise-control-center:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
+  dataVolumeCapacity: 10Gi
+  tls:
+    autoGeneratedCerts: true
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka:9071
+      tls:
+        enabled: true
+    connect:
+      - name: connect
+        url:  https://connect:8083
+        tls:
+          enabled: true
+    ksqldb:
+      - name: ksqldb
+        url:  https://ksqldb:8088
+        tls:
+          enabled: true
+    schemaRegistry:
+      url: https://schemaregistry:8081
+      tls:
+        enabled: true

kustomize/base/confluent/kafka.yaml

@@ -5,6 +5,23 @@ metadata:
 spec:
   replicas: 3
   image:
-    application: confluentinc/cp-server-operator:6.1.1.0
-    init: confluentinc/cp-init-container-operator:6.1.1.0
+    application: confluentinc/cp-server:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
   dataVolumeCapacity: 10Gi
+  configOverrides:
+    server:
+      - confluent.schema.registry.url=https://schemaregistry:8081
+  tls:
+    autoGeneratedCerts: true
+  listeners:
+    internal:
+      tls:
+        enabled: true
+    external:
+      tls:
+        enabled: true
+  dependencies:
+    zookeeper:
+      endpoint: zookeeper:2182
+      tls:
+        enabled: true

kustomize/base/confluent/ksqldb/ksqldb.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: platform.confluent.io/v1beta1
+kind: KsqlDB
+metadata:
+  name: ksqldb
+spec:
+  replicas: 1
+  image:
+    application: confluentinc/cp-ksqldb-server:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
+  dataVolumeCapacity: 10Gi
+  tls:
+    autoGeneratedCerts: true
+  dependencies:
+    kafka:
+      bootstrapEndpoint: kafka:9071
+      tls:
+        enabled: true

kustomize/base/confluent/ksqldb/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ksqldb.yaml
+

kustomize/base/confluent/kustomization.yaml

@@ -1,10 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - secrets
   - zookeeper.yaml
-  - kafka.yaml
-  - kafka-connect.yaml
-  - schema-registry.yaml
-  - control-centre.yaml
-  - ksqldb.yaml
-  - ca-pair-sslcerts.yaml
+  - kafka.yaml

kustomize/base/confluent/schmea-registry/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - schema-registry.yaml
+

kustomize/environments/sandbox/schema-registry.yaml → kustomize/base/confluent/schmea-registry/schema-registry.yaml

@@ -3,10 +3,14 @@ kind: SchemaRegistry
 metadata:
   name: schemaregistry
 spec:
+  replicas: 1
+  image:
+    application: confluentinc/cp-schema-registry:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
   tls:
     autoGeneratedCerts: true
   dependencies:
     kafka:
-      bootstrapEndpoint: kafka.sandbox.svc.cluster.local:9071
+      bootstrapEndpoint: kafka:9071
       tls:
         enabled: true

kustomize/base/confluent/secrets/ca-pair-sslcerts.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+data:
+  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURtRENDQW9DZ0F3SUJBZ0lVSEZEYWdiZ0Y4bTg0NzhoUEJpTktGOGI5MHVzd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1pERVJNQThHQTFVRUJoTUlWVzVwZG1WeWMyVXhEekFOQmdOVkJBZ1RCbEJoYm1kbFlURU9NQXdHQTFVRQpCeE1GUldGeWRHZ3hEVEFMQmdOVkJBb1RCRUZqYldVeERqQU1CZ05WQkFzVEJVMTVUM0puTVE4d0RRWURWUVFECkV3WlVaWE4wUTBFd0hoY05NakV3T0RFMk1USXhPREF3V2hjTk1qWXdPREUxTVRJeE9EQXdXakJrTVJFd0R3WUQKVlFRR0V3aFZibWwyWlhKelpURVBNQTBHQTFVRUNCTUdVR0Z1WjJWaE1RNHdEQVlEVlFRSEV3VkZZWEowYURFTgpNQXNHQTFVRUNoTUVRV050WlRFT01Bd0dBMVVFQ3hNRlRYbFBjbWN4RHpBTkJnTlZCQU1UQmxSbGMzUkRRVENDCkFTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBSzZ6NzJNbithSElYZ1RkODVqdi8rMm4KT0daQU9ITWhOQ2J2V1Y1cWVlTlgwaXZIZVhtRkF4TmFSc2ZKbGhOUllpTVRLRTRoUjZreTdEUWxFbVZLQ1g0cwpnaExYZ28wU3pUV3BhNkltcmFIbnplQWhob1gyaWxFdjRvUVVxMVFDdHhRd0NhVmxLYXhCM1pLNExzVVFuZlRqClBFL3JtQURwd2UxSjlteWxLeDJuNEl1V2NNRjE5dlBFdjc4dTdXU0FDNUQzdXVjTUwwRzByZys3K0srTXRQV1EKSk9UcWp2MHR6RGt4UXg0NmVlL042eFI0czdKbVRHc29SeC9KRDVOVUozNTJYYTFrNHhSUVltRnJ0U0V4UTBPUgpWSVNhYnQ2ZzA5L1NqQ0JrQTRWb0NoazBXL0o0K0k1VGR0clh0ZUpaTVBrWlhKMkxvS3hXM0NCVW0vRG9xUXNDCkF3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0VHTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME8KQkJZRUZOSFRzcEFlcURDVFUwMDB5SGU0ZzlEMURrcXNNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUF3VFErOApzTExxbURpV2RSeTZrOThyWWpPR3RXVW10ZWVSNDhNRWxQR3RSZ3BLWFhjcTBaSnd5ZHd3VTVZKzhuL3lyYW5UCmQzUWxEOWR5MDlZNHhYbDFkR2FhbWw0ZXE4cFZWQm04WE9kVkNaWnR3WjNHQ0I1NFdrQ2RHdnFIS3hnclVzb1IKMC9UYU15TVNXbmZMdk9XNHhLOXpQanAxYUc1eDdGeFJybW5OVjRUTHE5UFpOa3krK3Y2Y2dyNTF4OXdPOVNHaQpGTHBJOE9yTkdVeWxuZGhVMGFYSzRnZUFwZXh2cEJDNjdqY1Y4Nmo4emxKNkJaMnRRQXA0ZDFBeGVVQjROdmVsCmg3TFRCS0toemRER2lWUTFWYStoR1pGaE1GeXUxTzUrbVhzaUFwOTc1UEZ2S1pLMWJaeC90WjdMZjRSdXZpUjgKRHBQRUpMT2c1TXpQb2FFLwotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcnJQdll5ZjVvY2hlQk4zem1PLy83YWM0WmtBNGN5RTBKdTlaWG1wNTQxZlNLOGQ1CmVZVURFMXBHeDhtV0UxRmlJeE1vVGlGSHFUTHNOQ1VTWlVvSmZpeUNFdGVDalJMTk5hbHJvaWF0b2VmTjRDR0cKaGZhS1VTL2loQlNyVkFLM0ZEQUpwV1VwckVIZGtyZ3V4UkNkOU9NOFQrdVlBT25CN1VuMmJLVXJIYWZnaTVadwp3WFgyODhTL3Z5N3RaSUFMa1BlNjV3d3ZRYlN1RDd2NHI0eTA5WkFrNU9xTy9TM01PVEZESGpwNTc4M3JGSGl6CnNtWk1heWhISDhrUGsxUW5mblpkcldUakZGQmlZV3UxSVRGRFE1RlVoSnB1M3FEVDM5S01JR1FEaFdnS0dUUmIKOG5qNGpsTjIydGUxNGxrdytSbGNuWXVnckZiY0lGU2I4T2lwQ3dJREFRQUJBb0lCQUg1T2pTYXRRaWdiaXc0Tgp3ZzZ5aGVBMlkrcFpSL2JFN01XeGluVmVtZDBxSTBPUHBsMmJNd3dNMnR1YS9TdmVUUXQ2MEVzY2VJajF6aHd0CktVMmQ1NXNYWnJ0a29IaUtQVjhzckNyQnVBczBZbjVHL3RGZjI4U3kwQndhTFh4QnNrZWdLM2t1MktBM3hwQkwKSlZWSW5FcnU5eTM0OFI3SDRuTkNaQzFNaFEyY25SeFluc0Y0c2JxYWtvVHhMa08waTh3OXIwSXIwKzZIK0tJLwpjeUFWaWJuUi80L2VxODh0UDRhT0RrazZ0cUsweTk2S2ZqTEdpbGdsWW9CemUyK1AvNHhrSHlJYXZHd3ZCdllNCkgrc042OGhNbnhXbzg0cVp1RENROWJkOTA3MFV2d1NOc2ZNNkRBL3NBYWVTb0VSeHE4Z0xpSlhXZVZXQ3JNKzEKMFdrQTZJRUNnWUVBNXR4dStTZk0ra2NtM1hoMitKUWlIeERVNC9oankvYlBpV2k2d3ZjR2tFVksvRGQxZHlWQwprYUVjT05oblBFNytCZ21YbWVQdno3dG9nTmtrK2FncGY4L1JQVmFvNUQxemZSNkZESGU5QnBWWjRxUDdzYzk4CjFHRC9vVzFDaWVvUmVyQVlMM1BVWktIZ0g1a0ltR21ZR3owSUVxa0ZzeHl5bG4vVkc1T2ZoT0VDZ1lFQXdib0QKOUJKWGwxK3dUVHdPRE9KclFzZlFUc0lxR1pzZUtMYlM3cGwvT29mUWxCd0ZYN0VrV0RFejE4ZjV1MUEvUnhaSApCeUNTY2tiZW5GbXhFR0Zpa2Foc05EaGhzRWw3blpveDlDcWJFWXJwY08rUWRTTmlSSDBralpLdTladnd6VERxCmhMclV4b2Irbm5WdFFqenZCNmYrZThCc052UkJIY1hxUkhQZy8yc0NnWUFmaHY4SXU2eThpVGIvaTlJRGhJZGMKWXJvUUgvM1Evdlc4dURhNUpXOGFDTHlDMFRnVGUxMHA4aXllZnI4cWQwY3RuK1c2T0d5cG05aGJzRXZpWkZxMgpmN1Q5Z1hZekpVaytUd3F3ZHpHelVuWWlBMjcrQzIzcXV3eHVDVFBMRGRJeFJSY29jOU1Qb3lPdWJRSEZmcll0CkVocW1JYnNMaU9IQ3Q5WnNtNjV2UVFLQmdEa0tyWmxUUVVsSUdxQlRPNHZkcTFtSmZqT0NobVR0d0s2YVRuZUUKNk9weURSOWZOTkhkVmVkRjdZN3V5Z0MyZXp0U091M0ZVSG1ubG1xUkdrckpjZjJUaXZEQlhUcE1HTUxxSVNWVQoyajRMR2Z0di9xZ1BHQ00wZWV2WnNmeU53eFNYLzZXL1lXRm5aRmVDR2Y2ejlHbmJ0Rm41NmFNd3ZtK20wL2o5CjZkVXJBb0dCQUt2a0UrcnQ4MldRbjQydTNaei9aa3RMNHhLM3FKU3lDR0kycnpPQUtDTm92M1FPWlpiVVdqTDAKaTBYeVA5dFRDMnJYRDhZelVUblYzbUM5YjlGUkI5UFZ4OHY0OTllQjJaYTNiY29CRERpVU9VT0NsTlc5VElqWgpFNnFCSm5aY21QTllqZXl3UFE3aFFhbU1NUnA1MWlOeUVZYTVGOXFuU3hhcUN1cjRsaUN6Ci0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
+kind: Secret
+metadata:
+  name: ca-pair-sslcerts
+type: kubernetes.io/tls

kustomize/base/confluent/secrets/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ca-pair-sslcerts.yaml
+  - mds-public.yaml

kustomize/base/confluent/secrets/mds-public.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+data:
+  mdsPublicKey.pem: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF3TXlqblA0cWZkVEtoQ1M1c1BiVgpxaVhWeVExNXdyZVZBc0VxRXNuTUZ0Mkp0TUwxM0VMT1Eyc3pXbjU3V3p1NzgyYnlFdFlGbEYzVG9WVzNjbDRkCk9KUnphU0VRNnhlMTBSL2k3VG5lSXRFUWZwSnIvMkw0YnVidVFSR05lL0tyTE1FMGl2cjl1NElFYmJSUytsdHUKNkE5Z2d6R2NhRFN4Vi9leUtNTlBhZEhRL0FONEJaaWpBZUtaY0RUano2YkhqSjZFUTNZTmdxeW44NDZyZVFrOQpUb0habDhiR0hPaHo1Qzd5b0lmc3haZ1lIbG54NkpHc2lVWjVQMzZXR2MzOFpJQi9tNDVvOGN2NGlmVVZQVUIwCklRUTlBaFlJNVp1TXJ4RHNSUERYMkdHNkU1YlcydnFEV3lxWE9ZN2NTb0k3QWlrRmR3QVRXNFJ2N2V1RUpVeXoKTndJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t
+kind: Secret
+metadata:
+  creationTimestamp: null
+  name: mds-public

kustomize/base/confluent/zookeeper.yaml

@@ -5,7 +5,9 @@ metadata:
 spec:
   replicas: 3
   image:
-    application: confluentinc/cp-zookeeper-operator:6.1.1.0
-    init: confluentinc/cp-init-container-operator:6.1.1.0
+    application: confluentinc/cp-zookeeper:7.1.0
+    init: confluentinc/confluent-init-container:2.3.0
   dataVolumeCapacity: 10Gi
   logVolumeCapacity: 10Gi
+  tls:
+    autoGeneratedCerts: true

kustomize/environments/kustomization.yaml

@@ -1,2 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
 resources:
 - sandbox
+- production

kustomize/environments/production/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: production
+resources:
+  - namespace.yaml
+  - ../../base
+  - ../../base/confluent/connect
+  - ../../base/confluent/control-centre.yaml

kustomize/environments/production/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: production

kustomize/environments/sandbox/kustomization.yaml

@@ -1,10 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
 namespace: sandbox
 resources:
-- ../../base
-patchesStrategicMerge:
-  - control-centre.yaml
-  - kafka.yaml
-  - kafka-connect.yaml
-  - ksqldb.yaml
-  - schema-registry.yaml
-  - zookeeper.yaml
+  - namespace.yaml
+  - ../../base
+  - ../../base/confluent/schmea-registry

kustomize/environments/sandbox/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: sandbox

kustomize/operator/confluent-operator-helm-release-confluent.yaml

@@ -11,7 +11,7 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta1
 kind: HelmRelease
 metadata:
   name: confluent
-  namespace: sandbox
+  namespace: confluent
 spec:
   interval: 1m
   chart:
@@ -32,6 +32,7 @@ spec:
     force: true
     cleanupOnFail: true
   values:
+    namespaced: false
     image:
       registry: docker.io
     name: confluent-operator-global