Deploy codabench using kubernetes

charts/Chart.yaml

@@ -0,0 +1,34 @@
+apiVersion: v2
+name: codabench-chart
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"
+
+dependencies:
+  - name: rabbitmq
+    version: "14.7.0"
+    repository: "oci://registry.cern.ch/kubeflow/charts"
+    condition: rabbitmq.enabled
+  - name: redis
+    version: "19.5.4"
+    repository: "oci://registry.cern.ch/kubeflow/charts"
+    condition: redis.enabled

charts/dockerfiles/Dockerfile

@@ -0,0 +1,43 @@
+# Stage 1: Node.js builder
+FROM node:10 AS builder
+
+# Setup volume
+WORKDIR /app
+
+# Install packages
+ADD package.json .
+RUN npm install
+
+# Copy all files and build
+COPY . .
+RUN export PATH=./node_modules/.bin:$PATH && npm run build-stylus && npm run build-riot && npm run concat-riot
+
+# Stage 2: Python/Django (identical to main Dockerfile)
+FROM python:3.9.20
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y gcc build-essential && rm -rf /var/lib/apt/lists/*
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1
+ENV PATH=$PATH:/root/.local/bin
+
+# Install Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+
+# Set work directory before copying files
+WORKDIR /app
+
+# Copy only dependency descriptors first (for caching)
+COPY pyproject.toml poetry.lock ./
+RUN poetry install
+
+# Copy the rest of the application code
+COPY . /app
+
+# Copy built files from builder stage
+COPY --from=builder /app /app
+
+RUN ./manage.py collectstatic --noinput

charts/dockerfiles/Dockerfile.compute_worker

@@ -0,0 +1,35 @@
+FROM --platform=linux/amd64 python:3.9
+
+# This makes output not buffer and return immediately, nice for seeing results in stdout
+ENV PYTHONUNBUFFERED=1
+
+# Install Docker
+RUN apt-get update && curl -fsSL https://get.docker.com | sh
+
+# Install kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
+    install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \
+    rm kubectl
+
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
+# Poetry location so future commands (below) work
+ENV PATH=$PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+RUN mkdir codabench
+WORKDIR /app/
+COPY ./compute_worker/ ./
+COPY ./compute_worker/pyproject.toml ./
+COPY ./compute_worker/poetry.lock ./
+RUN poetry install
+RUN pip install redis
+RUN pip install kubernetes
+
+ADD compute_worker .
+
+CMD celery -A compute_worker worker \
+    -l info \
+    -Q compute-worker \
+    -n compute-worker@%n \
+    --concurrency=1

charts/dockerfiles/Dockerfile.flower

@@ -0,0 +1,31 @@
+FROM python:3.9
+
+# PYTHONUNBUFFERED: Force stdin, stdout and stderr to be totally unbuffered. (equivalent to `python -u`)
+# PYTHONHASHSEED: Enable hash randomization (equivalent to `python -R`)
+# PYTHONDONTWRITEBYTECODE: Do not write byte files to disk, since we maintain it as readonly. (equivalent to `python -B`)
+ENV PYTHONUNBUFFERED=1 PYTHONHASHSEED=random PYTHONDONTWRITEBYTECODE=1
+
+# Get latest root certificates
+RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates
+
+# # Install the required packages
+RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
+# Poetry location so future commands (below) work
+ENV PATH $PATH:/root/.local/bin
+# Want poetry to use system python of docker container
+RUN poetry config virtualenvs.create false
+RUN poetry config virtualenvs.in-project false
+
+RUN poetry init --no-interaction
+
+RUN poetry add redis=3.0.1
+RUN poetry add flower=0.9.3
+RUN poetry add celery="<5.0.0"
+
+# Default port
+EXPOSE 5555
+
+# Run as a non-root user by default, run as user with least privileges.
+USER nobody
+
+ENTRYPOINT ["flower"]

charts/templates/app-state-pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Values.appState.pvcName }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ .Values.appState.storage }}
+  storageClassName: {{ .Values.appState.storageClass }}

charts/templates/compute-worker-rbac.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: compute-worker-sa
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: compute-worker-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["pods/exec", "pods/log"]
+    verbs: ["create", "get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: compute-worker-bind
+  namespace: {{ .Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: compute-worker-sa
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: compute-worker-role
+  apiGroup: rbac.authorization.k8s.io

charts/templates/compute_worker-deployment.yaml

@@ -0,0 +1,73 @@
+{{- range .Values.compute_worker.brokers }}
+{{- $isDefault := eq .name "default" }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: compute-worker{{ if not $isDefault }}-{{ .name }}{{ end }}
+  labels:
+    app: compute-worker
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: compute-worker
+  template:
+    metadata:
+      labels:
+        app: compute-worker
+    spec:
+      serviceAccountName: compute-worker-sa
+      containers:
+        - name: compute-worker
+          image: "{{ $.Values.compute_worker.image.repository }}:{{ $.Values.compute_worker.image.tag }}"
+          imagePullPolicy: {{ $.Values.compute_worker.image.pullPolicy | default "IfNotPresent" }}
+          command:
+            - bash
+            - -c
+            - >
+              watchmedo auto-restart -p '*.py' --recursive -- celery -A compute_worker worker -l info -Q compute-worker -n compute-worker{{ if not $isDefault }}-{{ .name }}{{ end }}@%n
+          workingDir: /app
+          env:
+            - name: USE_GPU
+              value: '{{ .gpu.enabled }}'
+            - name: RESOURCE_LIMITS
+              value: '{{ toJson .gpu.resourceLimits }}'
+            - name: NODE_SELECTOR
+              value: '{{ toJson .gpu.nodeSelector }}'
+            - name: NUMBER_OF_POD_CREATION_RETRIES
+              value: '{{ $.Values.compute_worker.podCreationRetries.numberOfRetries }}'
+            - name: SLEEP_TIME_BETWEEN_RETRIES
+              value: '{{ $.Values.compute_worker.podCreationRetries.sleepTimeBetweenRetries }}'
+            - name: USERID
+              value: '{{ $.Values.compute_worker.submissionPods.securityContext.runAsUser }}'
+            - name: GROUPID
+              value: '{{ $.Values.compute_worker.submissionPods.securityContext.runAsGroup }}'
+            - name: FSGROUP
+              value: '{{ $.Values.compute_worker.submissionPods.securityContext.fsGroup }}'
+            - name: COMPUTE_WORKER_LABELS
+              value: '{{ toJson $.Values.compute_worker.submissionPods.metadata.labels }}'
+            - name: BROKER_URL
+              value: "{{ if .url }}{{ .url }}{{ else }}pyamqp://{{ $.Values.env.RABBITMQ_DEFAULT_USER }}:{{ $.Values.env.RABBITMQ_DEFAULT_PASS }}@{{ $.Values.env.RABBITMQ_HOST }}:{{ $.Values.env.RABBITMQ_PORT }}//{{ end }}"
+            - name: CODALAB_IGNORE_CLEANUP_STEP
+              value: "1"
+            {{- range $key, $value := $.Values.env }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+            {{- end }}
+          resources:
+            {{- toYaml $.Values.compute_worker.resources | nindent 12 }}
+          volumeMounts:
+            - name: docker-socket
+              mountPath: /var/run/docker.sock
+            - name: codabench-storage
+              mountPath: /codabench
+      volumes:
+        - name: docker-socket
+          hostPath:
+            path: /var/run/docker.sock
+            type: Socket
+        - name: codabench-storage
+          persistentVolumeClaim:
+            claimName: {{ $.Values.compute_worker.volumes.pvcName }}
+{{- end }}

charts/templates/django-deployment.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: django
+spec:
+  replicas: {{ .Values.django.replicas }}
+  selector:
+    matchLabels:
+      app: django
+  template:
+    metadata:
+      labels:
+        app: django
+    spec:
+      containers:
+        - name: django
+          image: "{{ .Values.django.image.repository }}:{{ .Values.django.image.tag }}"
+          imagePullPolicy: {{ .Values.django.image.pullPolicy }}
+          command:
+            - bash
+            - -c
+            - >
+              python manage.py collectstatic --noinput &&
+              cd {{ .Values.django.workingDir }} &&
+              watchmedo auto-restart -p '*.py' --recursive --
+              gunicorn asgi:application -w {{ .Values.django.gunicorn.workers }} -k uvicorn.workers.UvicornWorker -b :{{ .Values.django.port }} --capture-output --log-level {{ .Values.django.gunicorn.logLevel }}
+          env:
+          {{- range $key, $value := .Values.env }}
+            - name: {{ $key }}
+              value: "{{ $value }}"
+          {{- end }}
+            - name: DATABASE_URL
+              value: "postgres://{{ .Values.db.username }}:{{ .Values.db.password }}@{{ .Values.db.host }}:{{ .Values.db.port }}/{{ .Values.db.name }}"
+          volumeMounts:
+            - name: app-state
+              mountPath: /app/app-state
+      volumes:
+        - name: app-state
+          persistentVolumeClaim:
+            claimName: {{ .Values.appState.pvcName }}

charts/templates/django-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: django
+spec:
+  selector:
+    app: django
+  ports:
+    - protocol: TCP
+      port: 8000
+      targetPort: 8000
+

charts/templates/flower-deployment.yaml

@@ -0,0 +1,27 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flower
+  labels:
+    app: flower
+spec:
+  replicas: {{ .Values.flower.replicas }}
+  selector:
+    matchLabels:
+      app: flower
+  template:
+    metadata:
+      labels:
+        app: flower
+    spec:
+      containers:
+      - name: flower
+        image: "{{ .Values.flower.image.repository }}:{{ .Values.flower.image.tag }}"
+        imagePullPolicy: {{ .Values.flower.image.pullPolicy }}
+        ports:
+        - containerPort: {{ .Values.flower.service.port }}
+        env:
+        - name: CELERY_BROKER_URL
+          value: "pyamqp://{{ .Values.env.RABBITMQ_DEFAULT_USER }}:{{ .Values.env.RABBITMQ_DEFAULT_PASS }}@{{ .Values.env.RABBITMQ_HOST }}:{{ .Values.env.RABBITMQ_PORT }}//"
+        - name: FLOWER_PORT
+          value: "{{ .Values.flower.service.port }}"

charts/templates/flower-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: flower
+  labels:
+    app: flower
+spec:
+  type: ClusterIP
+  ports:
+  - port: {{ .Values.flower.service.port }}
+    targetPort: 5555
+  selector:
+    app: flower

charts/templates/ingress.yaml

@@ -0,0 +1,7 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: sample-http-ingress
+spec:
+  {{- toYaml .Values.ingress.spec | nindent 2 }}
+

charts/templates/istio.yaml

@@ -0,0 +1,11 @@
+{{- if .Values.istio.enableVirtualService}}
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: codabench-vs
+spec:
+  {{- toYaml .Values.istio.spec | nindent 12 }}
+
+{{- else }}
+{{- end }}
+

charts/templates/shared-pvc.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Values.sharedJob.pvcName }}
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ .Values.sharedJob.storage }}
+  storageClassName: {{ .Values.sharedJob.storageClass }}
+