Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Better connection strings #152

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

Better connection strings #152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
503729f
config: enhance database connection configuration
egabancho Mar 3, 2025
cf9e856
config: enhance RabbitMQ connection configuration
egabancho Mar 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 111 additions & 62 deletions charts/invenio/templates/_helpers.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{- end -}}

####################### RabbitMQ password secret #######################
####################### RabbitMQ connection configuration #######################
{{/*
This template renders the name of the secret that stores the password for RabbitMQ.
*/}}
Expand All @@ -95,19 +95,17 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{- end -}}

########################## RabbitMQ username ##########################
{{/*
This template renders the username for accessing RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.username" -}}
{{- if .Values.rabbitmq.enabled }}
{{- required "Missing .Values.rabbitmq.auth.username" .Values.rabbitmq.auth.username -}}
{{- else }}
{{- required "Missing .Values.rabbitmqExternal.username" .Values.rabbitmqExternal.username -}}
{{- required "Missing .Values.rabbitmqExternal.username" (tpl .Values.rabbitmqExternal.username .) -}}
{{- end }}
{{- end -}}

########################## RabbitMQ password ##########################
{{/*
This template renders the password for accessing RabbitMQ.
*/}}
Expand All @@ -119,43 +117,61 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{- end -}}

########################## RabbitMQ AMQP port ##########################
{{/*
Get the database password secret name
*/}}
{{- define "invenio.rabbitmq.secretName" -}}
{{- if .Values.rabbitmq.enabled -}}
{{- required "Missing .Values.rabbitmq.auth.existingPasswordSecret" (tpl .Values.rabbitmq.auth.existingPasswordSecret .) -}}
{{- else -}}
{{- required "Missing .Values.rabbitmqExternal.existingSecret" (tpl .Values.rabbitmqExternal.existingSecret .) -}}
{{- end -}}
{{- end -}}

{{/*
Get the database password secret key
*/}}
{{- define "invenio.rabbitmq.secretKey" -}}
{{- if .Values.rabbitmq.enabled -}}
{{- required "Missing .Values.rabbitmq.auth.existingSecretPasswordKey" .Values.rabbitmq.auth.existingSecretPasswordKey -}}
{{- else -}}
{{- required "Missing .Values.rabbitmqExternal.existingSecretPasswordKey" .Values.rabbitmqExternal.existingSecretPasswordKey -}}
{{- end -}}
{{- end -}}

{{/*
This template renders the AMQP port number for RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.amqpPort" -}}
{{- if .Values.rabbitmq.enabled }}
{{- required "Missing .Values.rabbitmq.service.ports.amqp" .Values.rabbitmq.service.ports.amqp -}}
{{- required "Missing .Values.rabbitmq.service.ports.amqp" .Values.rabbitmq.service.ports.amqp | quote -}}
{{- else }}
{{- required "Missing .Values.rabbitmqExternal.amqpPort" .Values.rabbitmqExternal.amqpPort -}}
{{- required "Missing .Values.rabbitmqExternal.amqpPort" (tpl .Values.rabbitmqExternal.amqpPort .) | quote -}}
{{- end }}
{{- end -}}

####################### RabbitMQ management port #######################
{{/*
This template renders the management port number for RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.managementPort" -}}
{{- if .Values.rabbitmq.enabled }}
{{- required "Missing .Values.rabbitmq.service.ports.manager" .Values.rabbitmq.service.ports.manager -}}
{{- required "Missing .Values.rabbitmq.service.ports.manager" .Values.rabbitmq.service.ports.manager | quote -}}
{{- else }}
{{- required "Missing .Values.rabbitmqExternal.managementPort" .Values.rabbitmqExternal.managementPort -}}
{{- required "Missing .Values.rabbitmqExternal.managementPort" (tpl .Values.rabbitmqExternal.managementPort .) | quote -}}
{{- end }}
{{- end -}}

########################## RabbitMQ hostname ##########################
{{/*
This template renders the hostname for RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.hostname" -}}
{{- if .Values.rabbitmq.enabled }}
{{- include "common.names.fullname" .Subcharts.rabbitmq -}}
{{- else }}
{{- required "Missing .Values.rabbitmqExternal.hostname" .Values.rabbitmqExternal.hostname }}
{{- required "Missing .Values.rabbitmqExternal.hostname" (tpl .Values.rabbitmqExternal.hostname .) }}
{{- end }}
{{- end -}}

########################## RabbitMQ protocol ##########################
{{/*
This template renders the protocol for RabbitMQ.
*/}}
Expand All @@ -167,44 +183,46 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{- end -}}

########################## RabbitMQ vhost ##########################
{{/*
This template renders the vhost for RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.vhost" -}}
{{- if .Values.rabbitmq.enabled }}
{{- "" }}
{{- else }}
{{- required "Missing .Values.rabbitmqExternal.vhost" .Values.rabbitmqExternal.vhost }}
{{- required "Missing .Values.rabbitmqExternal.vhost" (tpl .Values.rabbitmqExternal.vhost .) }}
{{- end }}
{{- end -}}

########################## Celery broker URI ##########################
{{/*
This template renders the URI for connecting to RabbitMQ.
*/}}
{{- define "invenio.rabbitmq.uri" -}}
{{- $username := (include "invenio.rabbitmq.username" .) -}}
{{- $password := (include "invenio.rabbitmq.password" .) -}}
{{- $port := (include "invenio.rabbitmq.amqpPort" .) -}}
{{- $hostname := (include "invenio.rabbitmq.hostname" .) -}}
{{- $protocol := (include "invenio.rabbitmq.protocol" .) -}}
{{- $vhost := (include "invenio.rabbitmq.vhost" .) -}}
{{- printf "%s://%s:%s@%s:%v/%s" $protocol $username $password $hostname $port $vhost}}
{{- end -}}

########################### RabbitMQ API URI ###########################
{{/*
This template renders the URI for RabbitMQ's API endpoint.
RabbitMQ connection env section.
*/}}
{{- define "invenio.rabbitmq.apiUri" -}}
{{- $username := (include "invenio.rabbitmq.username" .) -}}
{{- $password := (include "invenio.rabbitmq.password" .) -}}
{{- $port := (include "invenio.rabbitmq.managementPort" .) -}}
{{- $hostname := (include "invenio.rabbitmq.hostname" .) -}}
{{- printf "http://%s:%s@%s:%v/api/" $username $password $hostname $port }}
{{- define "invenio.config.queue" -}}
{{- $uri := "$(INVENIO_AMQP_BROKER_PROTOCOL)://$(INVENIO_AMQP_BROKER_USER):$(INVENIO_AMQP_BROKER_PASSWORD)@$(INVENIO_AMQP_BROKER_HOST):$(INVENIO_AMQP_BROKER_PORT)/$(INVENIO_AMQP_BROKER_VHOST)" -}}
- name: INVENIO_AMQP_BROKER_USER
value: {{ include "invenio.rabbitmq.username" . }}
- name: INVENIO_AMQP_BROKER_HOST
value: {{ include "invenio.rabbitmq.hostname" . }}
- name: INVENIO_AMQP_BROKER_PORT
value: {{ include "invenio.rabbitmq.amqpPort" . }}
- name: INVENIO_AMQP_BROKER_VHOST
value: {{ include "invenio.rabbitmq.vhost" . }}
- name: INVENIO_AMQP_BROKER_PROTOCOL
value: {{ include "invenio.rabbitmq.protocol" . }}
- name: INVENIO_AMQP_BROKER_PASSWORD
{{- if or (and .Values.rabbitmq.enabled .Values.rabbitmq.auth.password) .Values.rabbitmqExternal.password }}
value: {{ include "invenio.rabbitmq.password" . | quote }}
{{- else }}
valueFrom:
secretKeyRef:
name: {{ include "invenio.rabbitmq.secretName" .}}
key: {{ include "invenio.rabbitmq.secretKey" .}}
{{- end }}
- name: INVENIO_BROKER_URL
value: {{ $uri}}
- name: INVENIO_CELERY_BROKER_URL
value: {{ $uri}}
{{- end -}}

######################### OpenSearch hostname #########################
{{/*
This template renders the hostname of the OpenSearch instance.
Expand All @@ -217,79 +235,110 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{- end -}}

######################### PostgreSQL username #########################
######################### PostgreSQL connection configuration #########################
{{/*
This template renders the username used for the PostgreSQL instance.
*/}}
{{- define "invenio.postgresql.username" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.auth.username" .Values.postgresql.auth.username -}}
{{/* NOTE: Specifying username explicitly like this is suboptmal. Would be desirable to refactor Invenio so it can take the postgres username as a spearate environment variable which we can populate dynamically from the secret. */}}
{{- required "Missing .Values.postgresql.auth.username" (tpl .Values.postgresql.auth.username .) -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.username" .Values.postgresqlExternal.username -}}
{{- required "Missing .Values.postgresqlExternal.username" (tpl .Values.postgresqlExternal.username .) -}}
{{- end -}}
{{- end -}}

######################### PostgreSQL password #########################
{{/*
This template renders the password used for the PostgreSQL instance.
In production environments we encourage you to use secrets instead.
*/}}
{{- define "invenio.postgresql.password" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.auth.password" .Values.postgresql.auth.password -}}
{{/* NOTE: Specifying password explicitly like this is suboptmal. Would be desirable to refactor Invenio so it can take the postgres password as a spearate environment variable which we can populate dynamically from the secret. */}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.password" .Values.postgresqlExternal.password -}}
{{- end -}}
{{- end -}}

######################### PostgreSQL hostname #########################
{{/*
Get the database password secret name
*/}}
{{- define "invenio.postgresql.secretName" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.auth.existingSecret" (tpl .Values.postgresql.auth.existingSecret .) -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.existingSecret" (tpl .Values.postgresqlExternal.existingSecret .) -}}
{{- end -}}
{{- end -}}

{{/*
Get the database password secret key
*/}}
{{- define "invenio.postgresql.secretKey" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.auth.secretKeys.userPasswordKey" .Values.postgresql.auth.secretKeys.userPasswordKey -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.existingSecretPasswordKey" .Values.postgresqlExternal.existingSecretPasswordKey -}}
{{- end -}}
{{- end -}}

{{/*
This template renders the hostname used for the PostgreSQL instance.
*/}}
{{- define "invenio.postgresql.hostname" -}}
{{- if .Values.postgresql.enabled -}}
{{- include "postgresql.v1.primary.fullname" .Subcharts.postgresql -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.hostname" .Values.postgresqlExternal.hostname -}}
{{- required "Missing .Values.postgresqlExternal.hostname" (tpl .Values.postgresqlExternal.hostname .) -}}
{{- end -}}
{{- end -}}

########################### PostgreSQL port ###########################
{{/*
This template renders the port number used for the PostgreSQL instance.
*/}}
{{- define "invenio.postgresql.port" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.primary.service.ports.postgresql" .Values.postgresql.primary.service.ports.postgresql -}}
{{- required "Missing .Values.postgresql.primary.service.ports.postgresql" (tpl .Values.postgresql.primary.service.ports.postgresql .) -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.port" .Values.postgresqlExternal.port -}}
{{- required "Missing .Values.postgresqlExternal.port" (tpl .Values.postgresqlExternal.port .) -}}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried installing this and got the following error:

Error: INSTALLATION FAILED: template: invenio/templates/worker-deployment.yaml:43:14: executing "invenio/templates/worker-deployment.yaml" at <include "invenio.config.database" .>: error calling include: template: invenio/templates/_helpers.tpl:326:12: executing "invenio.config.database" at <include "invenio.postgresql.port" .>: error calling include: template: invenio/templates/_helpers.tpl:300:91: executing "invenio.postgresql.port" at <.Values.postgresql.primary.service.ports.postgresql>: wrong type for value; expected string; got float64

Guessing this is where it fails?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can try quote port number,
row 326
value: {{ include "invenio.postgresql.port" . }}
expects string as it renders template.

row 302
convert to string is better.
` {{- required "Missing .Values.postgresqlExternal.port" ((tpl .Values.postgresqlExternal.port .) | toString) -}}

{{- end -}}
{{- end -}}

############################ Database name ############################
{{/*
This template renders the name of the database in PostgreSQL.
*/}}
{{- define "invenio.postgresql.databaseName" -}}
{{- define "invenio.postgresql.database" -}}
{{- if .Values.postgresql.enabled -}}
{{- required "Missing .Values.postgresql.auth.database" .Values.postgresql.auth.database -}}
{{- required "Missing .Values.postgresql.auth.database" (tpl .Values.postgresql.auth.database .) -}}
{{- else -}}
{{- required "Missing .Values.postgresqlExternal.databaseName" .Values.postgresqlExternal.databaseName -}}
{{- required "Missing .Values.postgresqlExternal.database" (tpl .Values.postgresqlExternal.database .) -}}
{{- end -}}
{{- end -}}

####################### SQLAlchemy database URI #######################
{{/*
This template renders the SQLAlchemy database URI.
Define database connection env section.
*/}}
{{- define "invenio.sqlAlchemyDbUri" -}}
{{- $username := include "invenio.postgresql.username" . -}}
{{- $password := include "invenio.postgresql.password" . -}}
{{- $hostname := include "invenio.postgresql.hostname" . -}}
{{- $port := include "invenio.postgresql.port" . -}}
{{- $databaseName := include "invenio.postgresql.databaseName" . -}}
{{- printf "postgresql+psycopg2://%s:%s@%s:%v/%s" $username $password $hostname $port $databaseName -}}
{{- define "invenio.config.database" -}}
- name: INVENIO_DB_USER
value: {{ include "invenio.postgresql.username" . }}
- name: INVENIO_DB_HOST
value: {{ include "invenio.postgresql.hostname" . }}
- name: INVENIO_DB_PORT
value: {{ include "invenio.postgresql.port" . }}
- name: INVENIO_DB_NAME
value: {{ include "invenio.postgresql.database" . }}
- name: INVENIO_DB_PROTOCOL
value: "postgresql+psycopg2"
- name: INVENIO_DB_PASSWORD
{{- if or (and .Values.postgresql.enabled .Values.postgresql.auth.password) .Values.postgresqlExternal.password }}
value: {{ include "invenio.postgresql.password" . | quote }}
{{- else }}
valueFrom:
secretKeyRef:
name: {{ include "invenio.postgresql.secretName" .}}
key: {{ include "invenio.postgresql.secretKey" .}}
{{- end }}
- name: INVENIO_SQLALCHEMY_DATABASE_URI
value: "$(INVENIO_DB_PROTOCOL)://$(INVENIO_DB_USER):$(INVENIO_DB_PASSWORD)@$(INVENIO_DB_HOST):$(INVENIO_DB_PORT)/$(INVENIO_DB_NAME)"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lindhe What do you think about something like this? It should somehow address #112

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow. Maybe. 👀 The logic with defining each part as a separate env is sound, no issues there. But composing a new env value based on other envs like this, I'm unfamiliar with. If it works as your code indicates, then yeah I think this looks very promising.

How does this work? Does Kubernetes support the $(ENV) syntax in string values, or does that happen inside the container? I feel lost as to why this works. 😅 I assumed you tried it and it seems to work?

Assuming it works, I really like this because I think it should be forwards-compatible with inveniosoftware/invenio-config#57 and/or inveniosoftware/invenio-app-rdm#2918 which is nice.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

# compose-env.yaml
---
apiVersion: v1
data:
  secret: c2VjcmV0 # secret
kind: Secret
metadata:
  name: compose-env
---
apiVersion: v1
kind: Pod
metadata:
  name: compose-env
spec:
  containers:
  - name: compose-env
    image: bash
    env:
    - name: SECRET
      valueFrom:
          secretKeyRef:
            name: compose-env
            key: secret
    - name: PLAIN
      value: plain
    - name: MESSAGE
      value: "$(PLAIN) - $(SECRET)"
    command: ["echo"]
    args: ["$(MESSAGE)"]
$ kubectl apply -f compose-env.yaml
secret/compose-env created
pod/compose-env created

$ kubectl logs compose-env
plain - secret

It does work 😅
https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#using-environment-variables-inside-of-your-config

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am not sure when it happens, in particular after reading this in the docs Variables making use of others defined in the same context must come later in the list..
I would think the interpolation happens at run time inside the container. This Environment section of the output of kubectl describe pod

     Environment:
      SECRET:   <set to the key 'secret' in secret 'compose-env'>  Optional: false
      PLAIN:    plain
      MESSAGE:  $(PLAIN) - $(SECRET)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds convincing! Wow, this is so powerful (and a bit confusing)! 😀

I think this would be a great first step towards handling connection credentials better!

{{- end -}}

{{/*
Expand Down
8 changes: 2 additions & 6 deletions charts/invenio/templates/install-init-job.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,8 @@ spec:
env:
- name: TZ
value: {{ required "Missing .Values.global.timezone" .Values.global.timezone }}
- name: INVENIO_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_CELERY_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_SQLALCHEMY_DATABASE_URI
value: {{ include "invenio.sqlAlchemyDbUri" . }}
{{- include "invenio.config.queue" . | nindent 8 }}
{{- include "invenio.config.database" . | nindent 8 }}
volumeMounts:
{{- range $key, $value := .Values.invenio.vocabularies }}
- name: vocabularies
Expand Down
8 changes: 2 additions & 6 deletions charts/invenio/templates/web-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,8 @@ spec:
env:
- name: TZ
value: {{ required "Missing .Values.global.timezone" .Values.global.timezone }}
- name: INVENIO_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_CELERY_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_SQLALCHEMY_DATABASE_URI
value: {{ include "invenio.sqlAlchemyDbUri" . }}
{{- include "invenio.config.queue" . | nindent 8 }}
{{- include "invenio.config.database" . | nindent 8 }}
{{- with .Values.web.extraEnvVars }}
{{- toYaml . | nindent 8 }}
{{- end }}
Expand Down
8 changes: 2 additions & 6 deletions charts/invenio/templates/worker-beat-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,8 @@ spec:
env:
- name: TZ
value: {{ required "Missing .Values.global.timezone" .Values.global.timezone }}
- name: INVENIO_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_CELERY_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_SQLALCHEMY_DATABASE_URI
value: {{ include "invenio.sqlAlchemyDbUri" . }}
{{- include "invenio.config.queue" . | nindent 8 }}
{{- include "invenio.config.database" . | nindent 8}}
{{- with .Values.workerBeat.extraEnvVars }}
{{- toYaml . | nindent 8 }}
{{- end }}
Expand Down
8 changes: 2 additions & 6 deletions charts/invenio/templates/worker-deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,8 @@ spec:
env:
- name: TZ
value: {{ required "Missing .Values.global.timezone" .Values.global.timezone }}
- name: INVENIO_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_CELERY_BROKER_URL
value: {{ include "invenio.rabbitmq.uri" . }}
- name: INVENIO_SQLALCHEMY_DATABASE_URI
value: {{ include "invenio.sqlAlchemyDbUri" . }}
{{- include "invenio.config.queue" . | nindent 10 }}
{{- include "invenio.config.database" . | nindent 10 }}
{{- with .Values.worker.extraEnvVars }}
{{- toYaml . | nindent 10 }}
{{- end }}
Expand Down
Loading
Loading