Merge pull request #39 from langchain-ai/infra/add-external-clickhous…

…e-support Add clickhouse external support
langchain-ai · Jan 12, 2024 · 7ee153d · 7ee153d
2 parents 2db5b4d + 9ded3ae
commit 7ee153d
Show file tree

Hide file tree

Showing 7 changed files with 91 additions and 18 deletions.
diff --git a/charts/langsmith/templates/_helpers.tpl b/charts/langsmith/templates/_helpers.tpl
@@ -104,6 +104,18 @@ the user or some other secret provisioning mechanism
 {{- end }}
 {{- end }}
 
+{{/*
+Name of the secret containing the secrets for clickhouse. This can be overriden by a secrets file created by
+the user or some other secret provisioning mechanism
+*/}}
+{{- define "langsmith.clickhouseSecretsName" -}}
+{{- if .Values.clickhouse.external.existingSecretName }}
+{{- .Values.clickhouse.external.existingSecretName }}
+{{- else }}
+{{- include "langsmith.fullname" . }}-clickhouse
+{{- end }}
+{{- end }}
+
 {{/*
 Template containing common environment variables that are used by several services.
 */}}
@@ -122,6 +134,36 @@ Template containing common environment variables that are used by several servic
     secretKeyRef:
       name: {{ include "langsmith.redisSecretsName" . }}
       key: connection_url
+- name: CLICKHOUSE_DB
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_db
+- name: CLICKHOUSE_HOST
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_host
+- name: CLICKHOUSE_PORT
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_port
+- name: CLICKHOUSE_NATIVE_PORT
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_native_port
+- name: CLICKHOUSE_USER
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_user
+- name: CLICKHOUSE_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: {{ include "langsmith.clickhouseSecretsName" . }}
+      key: clickhouse_password
 - name: LOG_LEVEL
   value: debug
 {{- if .Values.config.oauth.enabled }}

diff --git a/charts/langsmith/templates/backend/run-migrations.yaml b/charts/langsmith/templates/backend/run-migrations.yaml
@@ -64,7 +64,7 @@ spec:
           - -source
           - file://clickhouse/migrations
           - -database
-          - "clickhouse://{{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}:{{ .Values.clickhouse.containerNativePort }}?username=clickhouse_admin&password=password&database=default&x-multi-statement=true&x-migrations-table-engine=MergeTree"
+          - "clickhouse://$(CLICKHOUSE_HOST):$(CLICKHOUSE_NATIVE_PORT)?username=$(CLICKHOUSE_USER)&password=$(CLICKHOUSE_PASSWORD)&database=$(CLICKHOUSE_DB)&x-multi-statement=true&x-migrations-table-engine=MergeTree"
           - up
           env:
             {{- include "langsmith.commonEnv" . | nindent 12 }}

diff --git a/charts/langsmith/templates/clickhouse/config-map.yaml b/charts/langsmith/templates/clickhouse/config-map.yaml
@@ -9,14 +9,8 @@ metadata:
 data:
   users.xml: |
     <clickhouse>
-        <profiles>
-            <default>
-                <readonly>0</readonly>
-            </default>
-        </profiles>
         <users>
             <default>
-                <password></password>
                 <access_management>1</access_management>
                 <named_collection_control>1</named_collection_control>
                 <show_named_collections>1</show_named_collections>

diff --git a/charts/langsmith/templates/clickhouse/secrets.yaml b/charts/langsmith/templates/clickhouse/secrets.yaml
@@ -0,0 +1,26 @@
+{{- if not .Values.clickhouse.external.existingSecretName }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "langsmith.clickhouseSecretsName" .}}
+  labels:
+    {{- include "langsmith.labels" . | nindent 4 }}
+  annotations:
+    {{- include "langsmith.annotations" . | nindent 4 }}
+data:
+  {{- if .Values.clickhouse.external.enabled }}
+  clickhouse_user: {{ .Values.clickhouse.external.user | b64enc }}
+  clickhouse_password: {{ .Values.clickhouse.external.password | b64enc }}
+  clickhouse_host: {{ .Values.clickhouse.external.host | b64enc }}
+  clickhouse_port: {{ .Values.clickhouse.external.port | b64enc }}
+  clickhouse_native_port: {{ .Values.clickhouse.external.nativePort | b64enc }}
+  clickhouse_db: {{ .Values.clickhouse.external.database | b64enc }}
+  {{- else }}
+  clickhouse_user: {{ "default" | b64enc }}
+  clickhouse_password: {{ "password" | b64enc }}
+  clickhouse_host: {{ printf "%s-%s" (include "langsmith.fullname" .) .Values.clickhouse.name | b64enc }}
+  clickhouse_port: {{ "8123" | b64enc }}
+  clickhouse_native_port: {{ "9000" | b64enc }}
+  clickhouse_db: {{ "default" | b64enc }}
+  {{- end}}
+{{- end }}
diff --git a/charts/langsmith/templates/clickhouse/stateful-set.yaml b/charts/langsmith/templates/clickhouse/stateful-set.yaml
@@ -47,7 +47,7 @@ spec:
             - name: CLICKHOUSE_DB
               value: default
             - name: CLICKHOUSE_USER
-              value: clickhouse_admin
+              value: default
             - name: CLICKHOUSE_PASSWORD
               value: password
           ports:
@@ -83,6 +83,9 @@ spec:
           securityContext:
             {{- toYaml .Values.clickhouse.statefulSet.securityContext | nindent 12 }}
           volumeMounts:
+            {{- with .Values.clickhouse.statefulSet.volumeMounts }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
             - mountPath: /var/lib/clickhouse
               name: data
             - mountPath: /etc/clickhouse-server/users.d/users.xml
@@ -107,6 +110,9 @@ spec:
             items:
               - key: users.xml
                 path: users.xml
+      {{- with .Values.clickhouse.statefulSet.volumes }}
+          {{- toYaml . | nindent 8 }}
+      {{- end }}
   volumeClaimTemplates:
     - apiVersion: v1
       kind: PersistentVolumeClaim

diff --git a/charts/langsmith/templates/config-map.yaml b/charts/langsmith/templates/config-map.yaml
@@ -17,8 +17,3 @@ data:
   {{- end }}
   LANGCHAIN_ENV: "local_kubernetes"
   FF_CLICKHOUSE_ENABLED: "true"
-  CLICKHOUSE_DB: "default"
-  CLICKHOUSE_HOST: {{ include "langsmith.fullname" . }}-{{ .Values.clickhouse.name }}
-  CLICKHOUSE_PORT: "8123"
-  CLICKHOUSE_USER: "clickhouse_admin"
-  CLICKHOUSE_PASSWORD: "password"
diff --git a/charts/langsmith/values.yaml b/charts/langsmith/values.yaml
@@ -15,19 +15,19 @@ images:
   backendImage:
     repository: "docker.io/langchain/langchainplus-backend"
     pullPolicy: IfNotPresent
-    tag: "6b3a5a5"
+    tag: "a624057"
   frontendImage:
     repository: "docker.io/langchain/langchainplus-frontend-dynamic"
     pullPolicy: IfNotPresent
-    tag: "d8abf62"
+    tag: "a624057"
   hubBackendImage:
     repository: "docker.io/langchain/langchainhub-backend"
     pullPolicy: IfNotPresent
-    tag: "d8abf62"
+    tag: "a624057"
   playgroundImage:
     repository: "docker.io/langchain/langchainplus-playground"
     pullPolicy: IfNotPresent
-    tag: "d8abf62"
+    tag: "a624057"
   postgresImage:
     repository: "docker.io/postgres"
     pullPolicy: IfNotPresent
@@ -123,9 +123,17 @@ backend:
 
 clickhouse:
   name: "clickhouse"
-  # External clickhouse is currently not supported.
   external:
+    # If enabled, use the following values to connect to an external database. This will also disable the
+    # creation of a clickhouse stateful-set and service.
     enabled: false
+    host: ""
+    port: "8123"
+    nativePort: "9000"
+    user: "default"
+    password: "password"
+    database: "default"
+    existingSecretName: ""
   containerHttpPort: 8123
   containerNativePort: 9000
   statefulSet:
@@ -137,6 +145,8 @@ clickhouse:
     nodeSelector: {}
     tolerations: []
     affinity: {}
+    volumes: []
+    volumeMounts: []
     persistence:
       size: 8Gi
       storageClassName: ""