DEVOPS-2707-added-keycloak-events-queue-in-rabbitmq

chart/templates/backend-deployment.yaml

@@ -303,7 +303,7 @@ spec:
             - name: SPRING_RABBITMQ_PORT
               value: {{ .Values.general.mq.port | quote }}
             - name: TRACKING_MIXPANEL_QUEUE_NAME
-              value: {{ .Values.general.mq.queue_name | quote }}
+              value: {{ include "lightrun-mq.getQueueNameByPrefix" (dict "prefix" "mixpanel-events" "Values" .Values) | quote }}
             {{- end }}
             {{- if .Values.deployments.backend.extraEnvs }}
               {{- toYaml .Values.deployments.backend.extraEnvs | nindent 12 }}
@@ -322,7 +322,7 @@ spec:
               value: "{{ include "http.scheme" . }}://{{ include "data_streamer.name" . }}:8080/events/post"
             {{- end }}
 
-      # waiting for mysql and keycloak initialization
+      # waiting for mysql, rabbitmq and keycloak initialization
       initContainers:
       {{- include "lightrun-backend.initContainer.download-async-profiler" . | nindent 6 }}
 
@@ -345,39 +345,7 @@ spec:
             value: {{ include "http.scheme" . }}://{{ include "lightrun-keycloak.name" . }}:9000/auth/health/started {{ if .Values.general.internal_tls.enabled }}--no-check-certificate{{ end }}
 
       {{ if .Values.general.mq.enabled }}
-      - name: wait-for-rabbitmq
-        image: "{{ .Values.deployments.backend.initContainers.wait_for_rabbitmq.image.repository }}:{{ .Values.deployments.backend.initContainers.wait_for_rabbitmq.image.tag }}"
-        imagePullPolicy: {{ .Values.deployments.backend.initContainers.wait_for_rabbitmq.image.pullPolicy }}
-        securityContext: {{ include "lightrun-be.containerSecurityContext" . | indent 10 }}
-        command:
-          - sh
-          - /scripts/wait-for-200.sh
-        resources:
-          limits:
-            memory: "100Mi"
-            cpu: "100m"
-          requests:
-            memory: "100Mi"
-            cpu: "100m"
-        env:
-          - name: MY_POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: AUTH_USER
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "secrets.backend.name" . }}
-                key: SPRING_RABBITMQ_USERNAME
-          - name: AUTH_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: {{ include "secrets.backend.name" . }}
-                key: SPRING_RABBITMQ_PASSWORD
-          - name: RABBITMQ_TCP_PORT
-            value: "15672"
-          - name: URL
-            value: {{ include "http.scheme" . }}://{{ include "lightrun-mq.endpoint" . }}:$(RABBITMQ_TCP_PORT)/api/overview {{ if .Values.general.internal_tls.enabled }}--no-check-certificate{{ end }}
+      {{- include "lightrun-mq.initContainer.wait-for-rabbitmq" (merge (dict "imageConfig" .Values.deployments.backend.initContainers.wait_for_rabbitmq "securityContext" "lightrun-be.containerSecurityContext") .) | nindent 6 }}
       {{- end }}
       - name: p12-creator
         image: "{{ .Values.deployments.backend.initContainers.p12_creator.image.repository }}:{{ .Values.deployments.backend.initContainers.p12_creator.image.tag }}"

chart/templates/helpers/_helpers.tpl

@@ -703,4 +703,32 @@ Container SecurityContext of lightrun artifacts
 {{- end -}}
 {{- end -}}
 
+{{/*
+Get queue name by prefix from the queue_names list.
+
+Parameters:
+- prefix: The prefix to match against queue names
+- Values: The chart values context
 
+Returns:
+- The first queue name that matches the prefix
+- Fails if no matching queue is found
+*/}}
+{{- define "lightrun-mq.getQueueNameByPrefix" -}}
+{{- $queue_names := .Values.general.mq.queue_names | default (list .Values.general.mq.queue_name) }}
+{{- if not $queue_names }}
+{{- fail "No queue names defined in general.mq.queue_names or general.mq.queue_name" }}
+{{- end }}
+{{- $prefix := .prefix | default "" | lower }}
+{{- $matching_queue := "" }}
+{{- range $queue_names }}
+{{- if and . (hasPrefix $prefix (. | lower)) }}
+{{- $matching_queue = . }}
+{{- break }}
+{{- end }}
+{{- end }}
+{{- if eq $matching_queue "" }}
+{{- fail (printf "No queue found with prefix '%s' in queue_names list" $prefix) }}
+{{- end }}
+{{- $matching_queue }}
+{{- end }}

chart/templates/helpers/_rabbitmq_initContainer.tpl

@@ -0,0 +1,40 @@
+{{/*
+Create a wait-for-rabbitmq init container
+*/}}
+{{- define "lightrun-mq.initContainer.wait-for-rabbitmq" -}}
+{{- $imageConfig := .imageConfig -}}
+{{- $securityContext := .securityContext -}}
+- name: wait-for-rabbitmq
+  image: "{{ $imageConfig.image.repository }}:{{ $imageConfig.image.tag }}"
+  imagePullPolicy: {{ $imageConfig.image.pullPolicy }}
+  securityContext: {{ include $securityContext . | indent 10 }}
+  command:
+    - sh
+    - /scripts/wait-for-200.sh
+  resources:
+    limits:
+      memory: "100Mi"
+      cpu: "100m"
+    requests:
+      memory: "100Mi"
+      cpu: "100m"
+  env:
+    - name: MY_POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: AUTH_USER
+      valueFrom:
+        secretKeyRef:
+          name: {{ include "secrets.backend.name" . }}
+          key: SPRING_RABBITMQ_USERNAME
+    - name: AUTH_PASSWORD
+      valueFrom:
+        secretKeyRef:
+          name: {{ include "secrets.backend.name" . }}
+          key: SPRING_RABBITMQ_PASSWORD
+    - name: RABBITMQ_TCP_PORT
+      value: "15672"
+    - name: URL
+      value: {{ include "http.scheme" . }}://{{ include "lightrun-mq.endpoint" . }}:$(RABBITMQ_TCP_PORT)/api/overview {{ if .Values.general.internal_tls.enabled }}--no-check-certificate{{ end }}
+{{- end -}} 

chart/templates/keycloak-deployment.yaml

@@ -170,6 +170,12 @@ spec:
               port: 9080
           {{- end }}
           env:
+            {{- if .Values.general.mq.enabled }}
+            - name: SPRING_RABBITMQ_HOST
+              value: {{ include "lightrun-mq.endpoint" . }}
+            - name: SPRING_RABBITMQ_PORT
+              value: {{ .Values.general.mq.port | quote }}
+            {{- end }}          
             - name: INFO_DEPLOYMENT
             {{ if eq .Values.general.deployment_type "saas" }}
               value: "SaaS"
@@ -293,6 +299,9 @@ spec:
     }}
       initContainers:
       {{- include "lightrun-keycloak.initContainer.download-async-profiler" . | nindent 6 }}
+      {{ if .Values.general.mq.enabled }}
+      {{- include "lightrun-mq.initContainer.wait-for-rabbitmq" (merge (dict "imageConfig" .Values.deployments.keycloak.initContainers.wait_for_rabbitmq "securityContext" "lightrun-keycloak.containerSecurityContext") .) | nindent 6 }}
+      {{- end }}      
     {{- end -}}
     {{- if .Values.general.readOnlyRootFilesystem }}
         {{- include "lightrun-keycloak.initContainer.read-only-rootfs" . }}

chart/templates/rabbitmq-cm.yaml

@@ -45,28 +45,32 @@ data:
         }    
       ],
       "queues": [
+        {{- $queue_names := .Values.general.mq.queue_names | default (list .Values.general.mq.queue_name) }}
+        {{- $last_index := sub (len $queue_names) 1 }}
+        {{- range $index, $queue_name := $queue_names }}
         {
-          "name": "{{ .Values.general.mq.queue_name }}.dlq",
+          "name": "{{ $queue_name }}.dlq",
           "vhost": "/",
           "durable": true,
           "auto_delete": false,
           "arguments": {
             "x-queue-type": "quorum",
             "x-created-by": "lightrun-onprem-server-chart"
           }
-        },    
-         {
-          "name": {{ .Values.general.mq.queue_name | quote }},
+        },
+        {
+          "name": {{ $queue_name | quote }},
           "vhost": "/",
           "durable": true,
           "auto_delete": false,
           "arguments": {
             "x-dead-letter-exchange": "",
-            "x-dead-letter-routing-key": "{{ .Values.general.mq.queue_name }}.dlq",
+            "x-dead-letter-routing-key": "{{ $queue_name }}.dlq",
             "x-queue-type": "quorum",
             "x-created-by": "lightrun-onprem-server-chart"
           }
-        }
+        }{{ if not (eq $index $last_index) }},{{ end }}
+        {{- end }}
       ]
     }
 

chart/values.yaml

@@ -79,20 +79,22 @@ general:
     ## Changing the queue name results in a new queue (and dlq) being created
     ## Old queues will not be deleted automatically
     ## New messages will be pushed and consumed from the new queue
-    queue_name: "mixpanel-events"
+    queue_names:
+      - "mixpanel-events"
+      - "keycloak-events"
     ## RabbitMQ Policy configuration
     ## Applies on all queues that match queue_regex_pattern
     policy:
-      queue_regex_pattern: "^mixpanel-events.*"
-      # The following configuration supports a queue size of up to 2000 messages,
+      queue_regex_pattern: "^.*-events.*"
+      # The following configuration supports queues size of up to 2000 messages,
       # where each message is 500 bytes, with a message TTL (time-to-live) of one week.
       #
-      # - message_ttl: Sets the time-to-live for messages in the queue to approximately 7 days (600,000,000 ms).
-      # - max_length: Limits the queue to a maximum of 2000 messages. New messages will be rejected if this limit is reached.
-      # - overflow: Configures the queue to "reject-publish," meaning new messages are rejected when the queue hits max_length.
-      # - max_length_bytes: Restricts the total size of all messages in the queue to 1,000,000 bytes (500 bytes × 2000 messages).
+      # - message_ttl: Sets the time-to-live for messages in the queues to approximately 7 days (600,000,000 ms).
+      # - max_length: Limits the queues to a maximum of 2000 messages. New messages will be rejected if this limit is reached.
+      # - overflow: Configures the queues to "reject-publish," meaning new messages are rejected when the queue hits max_length.
+      # - max_length_bytes: Restricts the total size of all messages in the queues to 1,000,000 bytes (500 bytes × 2000 messages).
       #
-      # This configuration ensures the queue does not exceed its size or time limits and avoids overloading the system.
+      # This configuration ensures the queues does not exceed its size or time limits and avoids overloading the system.
       message_ttl: 600000000
       max_length: 2000
       overflow: "reject-publish"
@@ -689,6 +691,11 @@ deployments:
           repository: lightruncom/chart-helper
           tag: "0.3.0-alpine-3.21.3-r0.lr-0"
           pullPolicy: ""
+      wait_for_rabbitmq:
+        image:
+          repository: lightruncom/chart-helper
+          tag: "0.3.0-alpine-3.21.3-r0.lr-0"
+          pullPolicy: ""          
     topologySpreadConstraints: []
     affinity: {}
     startupProbe: