Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
61224c8
fix: upgrade mlflow to 3.4.0 to address high-severity security vulner…
chris-sanders Sep 30, 2025
bd57950
fix: upgrade scikit-learn and use minimum version pinning for security
chris-sanders Sep 30, 2025
69c6dfe
fix: update container images to resolve CI failures
chris-sanders Sep 30, 2025
28ffdf3
fix: migrate to bitnamilegacy registry and bump chart versions
chris-sanders Sep 30, 2025
9674e62
fix: sync HelmChart versions and update KOTS registry references
chris-sanders Sep 30, 2025
7f377cb
fix: update wait-for-postgresql init container to use bitnamilegacy
chris-sanders Sep 30, 2025
157d5b0
fix: update mlflow main image to use bitnamilegacy
chris-sanders Sep 30, 2025
762f687
fix: remove --dev flag for MLflow 3.x compatibility
chris-sanders Sep 30, 2025
6d48e4b
fix: downgrade to MLflow 2.22.1 for compatibility
chris-sanders Sep 30, 2025
e813319
fix: upgrade to MLflow 3.3.2 with full compatibility fixes
chris-sanders Sep 30, 2025
43b7a89
fix: add MLFLOW_FLASK_SERVER_SECRET_KEY to CI test values
chris-sanders Sep 30, 2025
13d9880
fix: enable database upgrade for MLflow 3.x basic auth
chris-sanders Sep 30, 2025
4708577
fix: move MLFLOW_FLASK_SERVER_SECRET_KEY to container env
chris-sanders Sep 30, 2025
7df7833
fix: disable database upgrade, let MLflow auto-create auth tables
chris-sanders Sep 30, 2025
c23869d
fix: remove quotes from basic auth credentials in config
chris-sanders Sep 30, 2025
d074526
feat: upgrade MLflow to 3.3.2 and disable basic auth
chris-sanders Sep 30, 2025
157cc96
fix: create waitfor-postgres secret when postgres enabled, not just w…
chris-sanders Sep 30, 2025
d2afdff
fix: wait-for-postgresql should run when postgres enabled, not just b…
chris-sanders Oct 1, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion applications/mlflow/charts/infra/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.2.0
version: 0.2.1

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
Expand Down
4 changes: 2 additions & 2 deletions applications/mlflow/charts/infra/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ crdCheck:
# -- Image registry
registry: docker.io
# -- Image repository
repository: bitnami/kubectl
repository: bitnamilegacy/kubectl
# -- Image tag
tag: latest
tag: "1.33.4-debian-12-r0"
# -- CRDs to check
crds:
- name: tenants.minio.min.io
Expand Down
6 changes: 3 additions & 3 deletions applications/mlflow/charts/mlflow/Chart.lock
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
dependencies:
- name: replicated
repository: oci://registry.replicated.com/library
version: 1.5.1
digest: sha256:743ca58f2dbfd1408d98b10e27b95f55f5dff2cfc3020e14c707822a5d0f88e0
generated: "2025-04-16T12:26:22.509901-04:00"
version: 1.8.0
digest: sha256:3221c305cc2c7284ade24c125055434cd813f9107e05c36154668d2f7176055e
generated: "2025-09-30T15:07:20.581924-05:00"
4 changes: 2 additions & 2 deletions applications/mlflow/charts/mlflow/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@ apiVersion: v2
name: mlflow
description: A Helm chart for MLflow - Open source platform for the machine learning lifecycle.
type: application
version: "0.4.0"
appVersion: "2.10.0"
version: "0.5.1"
appVersion: "3.3.2"
home: https://github.com/mlflow/mlflow/tree/master/charts/mlflow
sources:
- https://github.com/mlflow/mlflow/tree/master/charts/mlflow
Expand Down
12 changes: 5 additions & 7 deletions applications/mlflow/charts/mlflow/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -93,17 +93,15 @@ spec:
{{- end }}
{{- end }}
initContainers:
{{- if .Values.mlflow.trackingServer.basicAuth.enabled }}
{{- if not .Values.mlflow.trackingServer.basicAuth.existingSecret }}
{{- if .Values.postgres.embedded.enabled }}
- name: wait-for-postgresql
image: docker.io/bitnami/postgresql:15.3.0-debian-11-r0
imagePullPolicy: {{ .Values.mlflow.image.pullPolicy }}
image: "{{ .Values.postgres.backup.image.registry }}/{{ .Values.postgres.backup.image.repository }}:{{ .Values.postgres.backup.image.tag }}"
imagePullPolicy: {{ .Values.postgres.backup.image.pullPolicy }}
command: ["sh", "-c", "until PGPASSWORD=$POSTGRES_PASSWORD psql -U $POSTGRES_USER -h $POSTGRES_HOST -p 5432 -d $POSTGRES_DB -c 'SELECT 1'; do sleep 1; done;"]
envFrom:
- secretRef:
name: {{ printf "%s-waitfor-postgres" (include "mlflow.fullname" .) | trunc 63 | trimAll "-" }}
{{- end }}
{{- end }}
{{- if .Values.mlflow.backendStore.databaseUpgrade }}
- name: mlflow-database-upgrade
image: {{ .Values.mlflow.image.registry | default "docker.io" }}/{{ .Values.mlflow.image.repository }}:{{ .Values.mlflow.image.tag | default (printf "v%s" .Chart.AppVersion) }}
Expand All @@ -129,7 +127,7 @@ spec:
name: {{ include "mlflow.fullname" . }}
{{- end }}
{{- with .Values.mlflow.extraInitContainers }}
{{ toYaml . | nindent 8 }}
{{ toYaml . | nindent 6 }}
{{- end }}
containers:
- name: {{ include "mlflow.fullname" . }}
Expand Down Expand Up @@ -235,7 +233,7 @@ spec:
{{- end }}
{{- end }}
{{- with .Values.mlflow.extraVolumes }}
{{- toYaml . | nindent 8 }}
{{- toYaml . | nindent 6 }}
{{- end }}
{{- with .Values.mlflow.hostAliases }}
hostAliases:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ stringData:
basic_auth.ini: |
[mlflow]
default_permission = {{ .defaultPermission }}
database_uri = {{ $dbUri }}
admin_username = {{ .adminUsername | quote }}
admin_password = {{ .adminPassword | quote }}
database_uri = sqlite:////tmp/basic_auth.db
admin_username = {{ .adminUsername }}
admin_password = {{ .adminPassword }}
authorization_function = {{ .authorizationFunction }}
{{- end }}
{{- end }}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
{{- if .Values.mlflow.trackingServer.basicAuth.enabled }}
{{- if not .Values.mlflow.trackingServer.basicAuth.existingSecret }}
{{- if .Values.postgres.embedded.enabled }}
apiVersion: v1
kind: Secret
metadata:
Expand All @@ -20,4 +19,3 @@ stringData:
POSTGRES_DB: {{ .Values.postgres.embedded.initdb.database | quote }}
{{- end }}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ spec:
defaultMode: 0777
initContainers:
- name: wait-for-postgresql
image: docker.io/bitnami/postgresql:15.3.0-debian-11-r0
imagePullPolicy: {{ .Values.mlflow.image.pullPolicy }}
image: "{{ .Values.postgres.backup.image.registry }}/{{ .Values.postgres.backup.image.repository }}:{{ .Values.postgres.backup.image.tag }}"
imagePullPolicy: {{ .Values.postgres.backup.image.pullPolicy }}
command:
[
"sh",
Expand All @@ -37,8 +37,8 @@ spec:
- secretRef:
name: {{ printf "%s-waitfor-postgres" (include "mlflow.fullname" .) | trunc 63 | trimAll "-" }}
- name: restore-db
image: docker.io/bitnami/postgresql:15.3.0-debian-11-r0
imagePullPolicy: {{ .Values.mlflow.image.pullPolicy }}
image: "{{ .Values.postgres.backup.image.registry }}/{{ .Values.postgres.backup.image.repository }}:{{ .Values.postgres.backup.image.tag }}"
imagePullPolicy: {{ .Values.postgres.backup.image.pullPolicy }}
command: ["/bin/sh", "/scripts/db-restore.sh"]
envFrom:
- secretRef:
Expand All @@ -50,7 +50,7 @@ spec:
mountPath: /scripts
containers:
- name: sleep
image: docker.io/bitnami/postgresql:15.3.0-debian-11-r0
image: "{{ .Values.postgres.backup.image.registry }}/{{ .Values.postgres.backup.image.repository }}:{{ .Values.postgres.backup.image.tag }}"
command: ["sh", "-c", "sleep infinity"]
envFrom:
- secretRef:
Expand Down
42 changes: 23 additions & 19 deletions applications/mlflow/charts/mlflow/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ mlflow:
# -- Image registry
registry: docker.io
# -- Image repository
repository: bitnami/mlflow
repository: bitnamilegacy/mlflow
# -- Image tag
tag: 2.12.2-debian-12-r1
tag: 3.3.2-debian-12-r0
# -- Image pull policy
pullPolicy: IfNotPresent
# -- Pod Labels for the mlflow deployment
Expand Down Expand Up @@ -73,7 +73,7 @@ mlflow:
# ENV_NAME_1: value
# ENV_NAME_2: value

# Extra environment variables in mlflow container
# Extra environment variables in mlflow container (not in init containers)
container: []
# - name: extra-env-name-1
# value: extra-env-value-1
Expand Down Expand Up @@ -103,19 +103,9 @@ mlflow:

# -- Extra volumes that can be mounted by containers belonging to the mlflow pod
extraVolumes: []
# - name: mlflow-volume
# persistentVolumeClaim:
# name: mlflow-pvc
# - name: mlflow-configmap-volume
# configMap:
# name: mlflow-configmap

# -- Extra volume mounts to mount into the mlflow container's file system
extraVolumeMounts: []
# - name: mlflow-volume
# mountPath: /opt/mlflow
# - name: mlflow-configmap-volume
# mountPath: /etc/mlflow

# -- Use hostAliases to add custom entries to /etc/hosts - mapping IP addresses to hostnames.
# [[ref]](https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/)
Expand Down Expand Up @@ -251,7 +241,7 @@ mlflow:
# -- Number of gunicorn worker processes to handle requests
workers: 1
# -- Extra arguments passed to the `mlflow server` command
extraArgs:
extraArgs: []
# A prefix which will be prepended to the path of all static paths
# - --static-prefix TEXT
# Additional command line options forwarded to gunicorn processes
Expand All @@ -260,14 +250,14 @@ mlflow:
# - --waitress-opts TEXT
# Path to the directory where metrics will be stored
# - --expose-prometheus /metrics
# If enabled, run the server with debug logging and auto-reload
- --dev
# Note: --dev flag cannot be used with --app-name in MLflow 3.x+

# Basic authentication configuration,
# for more information, please visit https://mlflow.org/docs/latest/auth/index.html#configuration
# NOTE: Basic auth is disabled due to compatibility issues with MLflow 3.x
basicAuth:
# -- Specifies whether to enable basic authentication
enabled: true
enabled: false
# -- Name of an existing secret which contains key `basic_auth.ini`
existingSecret: ""
# If enables BasicAuth and no existing secret is specified, creates a secret to store authentication configurations
Expand All @@ -276,14 +266,15 @@ mlflow:
defaultPermission: READ
# -- Default admin username if the admin is not already created
adminUsername: admin
# -- Default admin password if the admin is not already created
adminPassword: password
# -- Default admin password if the admin is not already created (min 12 chars for MLflow 3.x)
adminPassword: password123456
# -- Function to authenticate requests
authorizationFunction: mlflow.server.auth:authenticate_request_basic_auth

# For more information about how to configure backend store, please visit https://mlflow.org/docs/latest/tracking/backend-stores.html
backendStore:
# -- Specifies whether to run `mlflow db upgrade ${MLFLOW_BACKEND_STORE_URI}` to upgrade database schema when use a database as backend store
# MLflow 3.x with basic auth will auto-create auth tables on first run
databaseUpgrade: false
# -- Name of an existing secret which contains key `MLFLOW_BACKEND_STORE_URI`
# If an existing secret is not provided, a new secret will be created to store the backend store URI using the details from .Values.postgres when Embedded PostgreSQL is enabled
Expand Down Expand Up @@ -688,3 +679,16 @@ postgres:
port: 5432
# -- External Postgres database
database: mlflow

# -- Postgres backup configuration
backup:
# -- Image details for the postgres backup deployment
image:
# -- Image registry
registry: docker.io
# -- Image repository
repository: bitnamilegacy/postgresql
# -- Image tag
tag: "17.6.0-debian-12-r4"
# -- Image pull policy
pullPolicy: IfNotPresent
4 changes: 2 additions & 2 deletions applications/mlflow/release/infra-chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ metadata:
spec:
chart:
name: infra
chartVersion: 0.2.0
chartVersion: 0.2.1
exclude: 'repl{{ ConfigOptionEquals `postgres_type` `external_postgres` }}'
weight: -10
helmUpgradeFlags:
Expand All @@ -16,7 +16,7 @@ spec:
crdCheck:
image:
registry: repl{{ HasLocalRegistry | ternary LocalRegistryHost "docker.io" }}
repository: 'repl{{HasLocalRegistry | ternary LocalRegistryNamespace "bitnami" }}/kubectl'
repository: 'repl{{HasLocalRegistry | ternary LocalRegistryNamespace "bitnamilegacy" }}/kubectl'
cloudnative-pg:
image:
repository: '{{repl HasLocalRegistry | ternary LocalRegistryHost "ghcr.io" }}/{{repl HasLocalRegistry | ternary LocalRegistryNamespace "cloudnative-pg" }}/cloudnative-pg'
Expand Down
4 changes: 2 additions & 2 deletions applications/mlflow/release/mlflow-chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ metadata:
spec:
chart:
name: mlflow
chartVersion: 0.4.0
chartVersion: 0.5.1
weight: 10
helmUpgradeFlags:
- --wait
Expand All @@ -15,7 +15,7 @@ spec:
mlflow:
image:
registry: repl{{ HasLocalRegistry | ternary LocalRegistryHost "docker.io" }}
repository: 'repl{{HasLocalRegistry | ternary LocalRegistryNamespace "bitnami" }}/mlflow'
repository: 'repl{{HasLocalRegistry | ternary LocalRegistryNamespace "bitnamilegacy" }}/mlflow'
ingress:
enabled: true
className: repl{{ ConfigOption "mlflow_ingress_class_name"}}
Expand Down
7 changes: 5 additions & 2 deletions applications/mlflow/tests/helm/nodeport-ingress-disabled.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,19 @@
# These values specifically configure the service to use NodePort for testing

mlflow:
ingress:
ingress:
enabled: false
# Service configuration for MLflow
service:
# Use NodePort to expose the service on a specific port
type: NodePort
# Service port number (internal)
port: 5000
# Hardcoded nodePort for consistent access
# Hardcoded nodePort for consistent access
# Note: Must be between 30000-32767
nodePort: 30080
# Service port name
name: http
# Environment configuration
# env:
# container: []
20 changes: 8 additions & 12 deletions applications/mlflow/tests/mlflow_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ def check_server_connection(tracking_uri, timeout=30, retry_interval=5):
host = parsed_url.hostname
port = parsed_url.port or (443 if parsed_url.scheme == 'https' else 80)

# Authentication credentials
auth = ("admin", "password")
# Authentication disabled for MLflow 3.x compatibility
auth = None

start_time = time.time()
while time.time() - start_time < timeout:
Expand All @@ -65,8 +65,8 @@ def check_server_connection(tracking_uri, timeout=30, retry_interval=5):

# Then try an HTTP request to the root URL
try:
# For our test environment, always disable SSL verification and include auth
response = requests.get(health_url, timeout=5, verify=False, auth=auth)
# For our test environment, always disable SSL verification
response = requests.get(health_url, timeout=5, verify=False, auth=auth if auth else None)
status_code = response.status_code
logger.info(f"Server returned status code: {status_code}")

Expand Down Expand Up @@ -109,12 +109,8 @@ def run_mlflow_test(tracking_uri, connection_timeout=60):
logger.error("Failed to connect to MLflow server, aborting test")
return False

# Set MLflow tracking URI with authentication
# Format: http(s)://username:password@hostname:port
parsed_url = urlparse(tracking_uri)
auth_url = f"{parsed_url.scheme}://admin:password@{parsed_url.netloc}{parsed_url.path}"
logger.info(f"Using authenticated tracking URI")
mlflow.set_tracking_uri(auth_url)
# Set MLflow tracking URI
mlflow.set_tracking_uri(tracking_uri)

# Load the Iris dataset
logger.info("Loading dataset and training model...")
Expand Down Expand Up @@ -246,8 +242,8 @@ def main():
if args.protocol == "http":
logger.info("Using HTTP protocol (insecure)")

# Note about hardcoded credentials
logger.info("Using hardcoded authentication (admin/password)")
# Note: Authentication disabled for MLflow 3.x
logger.info("Authentication disabled (MLflow 3.x without basic auth)")

# Ensure dependencies are installed
ensure_dependencies()
Expand Down
13 changes: 6 additions & 7 deletions applications/mlflow/tests/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
setuptools>=65.0.0
wheel>=0.40.0
mlflow==2.11.0
# Pre-built wheels for problematic packages
numpy<2.0.0
pandas<2.2.0
scikit-learn<1.4.0
# Pin pyarrow to a version with pre-built wheels
pyarrow==15.0.0
mlflow>=3.4.0
# Minimum versions confirmed working
numpy>=1.24.0,<2.0.0
pandas>=2.0.0,<3.0.0
scikit-learn>=1.5.0
pyarrow>=15.0.0
requests>=2.31.0
urllib3>=2.0.0
Loading