From 547d1a4ebfbcbae7e1fe005c5531be5cfe51dba7 Mon Sep 17 00:00:00 2001 From: Ricardo Martinelli de Oliveira Date: Thu, 22 Jul 2021 10:32:25 -0300 Subject: [PATCH] Refactoring on Superset manifests (#459) --- ai-library/tests/setup/kfctl_openshift.yaml | 4 - kfdef/kfctl_openshift.yaml | 4 - .../kfctl_openshift_distributed_training.yaml | 4 - superset/README.md | 66 ++++++++++- superset/base/deployment.yaml | 63 ++++++++-- superset/base/kustomization.yaml | 110 +++++------------- superset/base/params.env | 30 ++--- superset/base/params.yaml | 8 +- superset/base/secret-superset.yaml | 14 +-- superset/base/secret.yaml | 6 +- superset/base/supersetdb-deployment.yaml | 17 ++- superset/base/supersetdb-secret.yaml | 6 +- tests/setup/kfctl_openshift.yaml | 2 +- 13 files changed, 180 insertions(+), 154 deletions(-) diff --git a/ai-library/tests/setup/kfctl_openshift.yaml b/ai-library/tests/setup/kfctl_openshift.yaml index 5c6de5e2b..3cceced4f 100644 --- a/ai-library/tests/setup/kfctl_openshift.yaml +++ b/ai-library/tests/setup/kfctl_openshift.yaml @@ -28,10 +28,6 @@ spec: # Note: In order to utilize ai-library, you also need to have Seldon installed name: ai-library-operator - kustomizeConfig: - parameters: - # Note: The admin username is admin - - name: SUPERSET_ADMIN_PASSWORD - value: admin repoRef: name: manifests path: superset diff --git a/kfdef/kfctl_openshift.yaml b/kfdef/kfctl_openshift.yaml index 30fa2396c..8e95756bb 100644 --- a/kfdef/kfctl_openshift.yaml +++ b/kfdef/kfctl_openshift.yaml @@ -16,10 +16,6 @@ spec: path: odhseldon/cluster name: odhseldon - kustomizeConfig: - parameters: - # Note: The admin username is admin - - name: SUPERSET_ADMIN_PASSWORD - value: admin repoRef: name: manifests path: superset diff --git a/kfdef/kfctl_openshift_distributed_training.yaml b/kfdef/kfctl_openshift_distributed_training.yaml index f08a8908a..bf5e6f429 100644 --- a/kfdef/kfctl_openshift_distributed_training.yaml +++ b/kfdef/kfctl_openshift_distributed_training.yaml @@ -28,10 +28,6 @@ spec: # Note: In order to utilize ai-library, you also need to have Seldon installed name: ai-library-operator - kustomizeConfig: - parameters: - # Note: The admin username is admin - - name: SUPERSET_ADMIN_PASSWORD - value: admin repoRef: name: manifests path: superset diff --git a/superset/README.md b/superset/README.md index 27d6f2f98..8527a8ea4 100644 --- a/superset/README.md +++ b/superset/README.md @@ -11,17 +11,73 @@ To install Superset add the following to the `kfctl` yaml file. ``` - kustomizeConfig: - parameters: - # Note: The admin username is admin - - name: SUPERSET_ADMIN_PASSWORD - value: admin repoRef: name: manifests path: superset name: superset ``` -By default the user and password to the Superset portal is admin/admin. You can change the password by changing the value of the `SUPERSET_ADMIN_PASSWORD`. To launch the portal, go to the routes in the namespace you installed Open Data Hub and click on the route with `superset` name. +By default the user and password to the Superset portal is admin/admin. To launch the portal, go to the routes in the namespace you installed Open Data Hub and click on the route with `superset` name. + +### Parameters + +There are 11 parameters exposed via KfDef. + +#### storage_class + +Name of the storage class to be used for PVC created by Hue's database. This requires `storage-class` **overlay** to be enabled as well to work. +#### superset_secret + +The secret containing the environment variables to set the admin credentials and the Superset app secret key used to encrypt information in the database. If not specified, environment variables from [`superset`](base/secret-superset.yaml) will be used. + +When creating a custom secret, the following information should be added: + +* **SUPERSET_ADMIN_USER**: The username of the Superset administrator. If using the `superset` secret, `admin` will be used +* **SUPERSET_ADMIN_FNAME**: The First Name of the Superset administrator. If using the `superset` secret, `admin` will be used +* **SUPERSET_ADMIN_LNAME**: The LastName of the Superset administrator. If using the `superset` secret, `admin` will be used +* **SUPERSET_ADMIN_EMAIL**: The e-mail of the Superset administrator. If using the `superset` secret, `admin@fab.org` will be used +* **SUPERSET_ADMIN_PASSWORD**: The password of the Superset administrator. If using the `superset` secret, `admin` will be used +* **SUPERSET_SECRET_KEY**: The app secret key used by Superset to encrypt information in the database. If using the `superset` secret, `thisISaSECRET_1234` will be used + +#### superset_db_secret + +This parameter configures the Superset database. The secret of choice must contain `database-name`, `database-user`, and `database-password` keys. If not set, credentials from [`supersetdb-secret`](base/supersetdb-secret.yaml) will be used instead. + +#### superset_memory_requests + +This parameter will configure the Memory request for Superset. If not set, the default value `1Gi` will be used instead. + +#### superset_memory_limits + +This parameter will configure the Memory limits for Superset. If not set, the default value `2Gi` will be used instead. + +#### superset_cpu_requests + +This parameter will configure the CPU request for Superset. If not set, the default value `300m` will be used instead. + +#### superset_cpu_limits + +This parameter will configure the CPU limits for Superset. If not set, the default value `2` will be used instead. + +#### superset_db_memory_requests + +This parameter will configure the Memory request for Superset Database. If not set, the default value `300Mi` will be used instead. + +#### superset_db_memory_limits + +This parameter will configure the Memory limits for Superset Database. If not set, the default value `1Gi` will be used instead. + +#### superset_db_cpu_requests + +This parameter will configure the CPU request for Superset Database. If not set, the default value `300m` will be used instead. + +#### superset_db_cpu_limits + +This parameter will configure the CPU request for Superset Database. If not set, the default value `1` will be used instead. + +### Superset config file customization + +Superset manifests comes with a [`superset-config`](base/secret.yaml) secret, which will configure basic parameters for Superset like the database SQLAlchemy URL to connect to its database. ### Superset Database Initialization diff --git a/superset/base/deployment.yaml b/superset/base/deployment.yaml index 956bc6959..3b85cd877 100644 --- a/superset/base/deployment.yaml +++ b/superset/base/deployment.yaml @@ -17,19 +17,58 @@ spec: spec: initContainers: - name: superset-init - image: $(SUPERSET_IMAGE) - command: ['sh', '-c', "sleep 30; superset-init --username $(SUPERSET_ADMIN_USER) --firstname $(SUPERSET_ADMIN_FNAME) --lastname $(SUPERSET_ADMIN_LNAME) --email $(SUPERSET_ADMIN_EMAIL) --password $(SUPERSET_ADMIN_PASSWORD)"] - imagePullPolicy: Always + env: + - name: SUPERSET_CONFIG_PATH + value: /etc/superset/superset_config.py + - name: POSTGRESQL_USERNAME + valueFrom: + secretKeyRef: + key: database-user + name: $(superset_db_secret) + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: $(superset_db_secret) + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: $(superset_db_secret) + envFrom: + - secretRef: + name: $(superset_secret) + image: superset + command: ['sh', '-c', "sleep 30; superset-init --username $SUPERSET_ADMIN_USER --firstname $SUPERSET_ADMIN_FNAME --lastname $SUPERSET_ADMIN_LNAME --email $SUPERSET_ADMIN_EMAIL --password $SUPERSET_ADMIN_PASSWORD"] volumeMounts: - mountPath: /etc/superset name: superset-config containers: - name: superset - envFrom: - - secretRef: - name: superset - image: $(SUPERSET_IMAGE) - imagePullPolicy: Always + env: + - name: SUPERSET_SECRET_KEY + valueFrom: + secretKeyRef: + key: SUPERSET_SECRET_KEY + name: $(superset_secret) + - name: POSTGRESQL_USERNAME + valueFrom: + secretKeyRef: + key: database-user + name: $(superset_db_secret) + - name: POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + key: database-password + name: $(superset_db_secret) + - name: POSTGRESQL_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: $(superset_db_secret) + - name: SUPERSET_CONFIG_PATH + value: /etc/superset/superset_config.py + image: superset ports: - containerPort: 8088 protocol: TCP @@ -38,11 +77,11 @@ spec: name: superset-config resources: requests: - cpu: $(SUPERSET_CPU_REQUESTS) - memory: $(SUPERSET_MEMORY_REQUESTS) + cpu: $(superset_cpu_requests) + memory: $(superset_memory_requests) limits: - cpu: $(SUPERSET_CPU_LIMITS) - memory: $(SUPERSET_MEMORY_LIMITS) + cpu: $(superset_cpu_limits) + memory: $(superset_memory_limits) volumes: - name: superset-config secret: diff --git a/superset/base/kustomization.yaml b/superset/base/kustomization.yaml index ebec08e99..e3f704f36 100644 --- a/superset/base/kustomization.yaml +++ b/superset/base/kustomization.yaml @@ -1,7 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: -- service-account.yaml - pvc.yaml - secret-superset.yaml - deployment.yaml @@ -25,143 +24,94 @@ configMapGenerator: - params.env vars: - - name: SUPERSET_ADMIN_PASSWORD + - name: superset_secret objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_ADMIN_PASSWORD - - name: SUPERSET_ADMIN_USER + fieldpath: data.superset_secret + - name: superset_db_secret objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_ADMIN_USER - - name: SUPERSET_ADMIN_FNAME + fieldpath: data.superset_db_secret + - name: superset_memory_requests objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_ADMIN_FNAME - - name: SUPERSET_ADMIN_LNAME + fieldpath: data.superset_memory_requests + - name: superset_memory_limits objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_ADMIN_LNAME - - name: SUPERSET_ADMIN_EMAIL + fieldpath: data.superset_memory_limits + - name: superset_cpu_requests objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_ADMIN_EMAIL - - name: SUPERSET_VOLUME_SIZE + fieldpath: data.superset_cpu_requests + - name: superset_cpu_limits objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_VOLUME_SIZE - - name: SUPERSET_SECRET_KEY + fieldpath: data.superset_cpu_limits + - name: superset_db_memory_requests objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_SECRET_KEY - - name: SUPERSET_DB_USERNAME + fieldpath: data.superset_db_memory_requests + - name: superset_db_memory_limits objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_DB_USERNAME - - name: SUPERSET_DB_PASSWORD + fieldpath: data.superset_db_memory_limits + - name: superset_db_cpu_requests objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_DB_PASSWORD - - name: LOG_LEVEL + fieldpath: data.superset_db_cpu_requests + - name: superset_db_cpu_limits objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.LOG_LEVEL - - name: SUPERSET_MEMORY_REQUESTS + fieldpath: data.superset_db_cpu_limits + - name: storage_class objref: name: superset-config kind: ConfigMap apiVersion: v1 fieldref: - fieldpath: data.SUPERSET_MEMORY_REQUESTS - - name: SUPERSET_MEMORY_LIMITS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_MEMORY_LIMITS - - name: SUPERSET_CPU_REQUESTS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_CPU_REQUESTS - - name: SUPERSET_CPU_LIMITS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_CPU_LIMITS - - name: SUPERSET_DB_MEMORY_REQUESTS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_DB_MEMORY_REQUESTS - - name: SUPERSET_DB_MEMORY_LIMITS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_DB_MEMORY_LIMITS - - name: SUPERSET_DB_CPU_REQUESTS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_DB_CPU_REQUESTS - - name: SUPERSET_DB_CPU_LIMITS - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_DB_CPU_LIMITS - - name: SUPERSET_IMAGE - objref: - name: superset-config - kind: ConfigMap - apiVersion: v1 - fieldref: - fieldpath: data.SUPERSET_IMAGE - + fieldpath: data.storage_class configurations: - params.yaml generatorOptions: disableNameSuffixHash: true + +images: + - name: superset + newName: quay.io/opendatahub/superset + newTag: "1.1.0" + - name: supersetdb + newName: quay.io/internaldatahub/postgresql-96-centos7 + newTag: "9.6" diff --git a/superset/base/params.env b/superset/base/params.env index 62256c652..02dfe094d 100644 --- a/superset/base/params.env +++ b/superset/base/params.env @@ -1,19 +1,11 @@ -SUPERSET_ADMIN_PASSWORD=admin -SUPERSET_ADMIN_USER=admin -SUPERSET_ADMIN_FNAME=admin -SUPERSET_ADMIN_LNAME=admin -SUPERSET_ADMIN_EMAIL=admin@fab.org -SUPERSET_VOLUME_SIZE=512Mi -SUPERSET_SECRET_KEY=thisISaSECRET_1234 -SUPERSET_DB_USERNAME=changeme -SUPERSET_DB_PASSWORD=changeme -LOG_LEVEL=INFO -SUPERSET_MEMORY_REQUESTS=1Gi -SUPERSET_MEMORY_LIMITS=2Gi -SUPERSET_CPU_REQUESTS=300m -SUPERSET_CPU_LIMITS=2 -SUPERSET_DB_MEMORY_REQUESTS=300Mi -SUPERSET_DB_MEMORY_LIMITS=1Gi -SUPERSET_DB_CPU_REQUESTS=300m -SUPERSET_DB_CPU_LIMITS=1 -SUPERSET_IMAGE=quay.io/opendatahub/superset:1.1.0 +storage_class=default +superset_secret=superset +superset_db_secret=supersetdb +superset_memory_requests=1Gi +superset_memory_limits=2Gi +superset_cpu_requests=300m +superset_cpu_limits=2 +superset_db_memory_requests=300Mi +superset_db_memory_limits=1Gi +superset_db_cpu_requests=300m +superset_db_cpu_limits=1 diff --git a/superset/base/params.yaml b/superset/base/params.yaml index 707de9553..416cd6095 100644 --- a/superset/base/params.yaml +++ b/superset/base/params.yaml @@ -14,7 +14,11 @@ varReference: kind: Deployment - path: spec/template/spec/containers[]/resources/limits/memory kind: Deployment - - path: spec/template/spec/containers[]/image + - path: spec/template/spec/containers[]/envFrom/secretRef/name kind: Deployment - - path: spec/template/spec/initContainers[]/image + - path: spec/template/spec/initContainers[]/envFrom/secretRef/name + kind: Deployment + - path: spec/template/spec/containers[]/env[]/valueFrom/secretKeyRef/name + kind: Deployment + - path: spec/template/spec/initContainers[]/env[]/valueFrom/secretKeyRef/name kind: Deployment diff --git a/superset/base/secret-superset.yaml b/superset/base/secret-superset.yaml index 89d36fb62..fcb6c435b 100644 --- a/superset/base/secret-superset.yaml +++ b/superset/base/secret-superset.yaml @@ -5,11 +5,9 @@ metadata: labels: app: superset stringData: - SUPERSET_ADMIN_USER: $(SUPERSET_ADMIN_USER) - SUPERSET_ADMIN_FNAME: $(SUPERSET_ADMIN_FNAME) - SUPERSET_ADMIN_LNAME: $(SUPERSET_ADMIN_LNAME) - SUPERSET_ADMIN_EMAIL: $(SUPERSET_ADMIN_EMAIL) - SUPERSET_ADMIN_PASSWORD: $(SUPERSET_ADMIN_PASSWORD) - SUPERSET_VOLUME_SIZE: $(SUPERSET_VOLUME_SIZE) - FLASK_APP: superset - SUPERSET_CONFIG_PATH: /etc/superset/superset_config.py + SUPERSET_ADMIN_USER: admin + SUPERSET_ADMIN_FNAME: admin + SUPERSET_ADMIN_LNAME: admin + SUPERSET_ADMIN_EMAIL: admin@fab.org + SUPERSET_ADMIN_PASSWORD: admin + SUPERSET_SECRET_KEY: thisISaSECRET_1234 diff --git a/superset/base/secret.yaml b/superset/base/secret.yaml index 13bde2fb5..8525f0d78 100644 --- a/superset/base/secret.yaml +++ b/superset/base/secret.yaml @@ -9,11 +9,11 @@ stringData: import os from flask_appbuilder.security.manager import AUTH_DB, AUTH_LDAP MAPBOX_API_KEY = os.getenv('MAPBOX_API_KEY', '') - SQLALCHEMY_DATABASE_URI = 'postgresql://$(SUPERSET_DB_USERNAME):$(SUPERSET_DB_PASSWORD)@supersetdb:5432/supersetdb' + SQLALCHEMY_DATABASE_URI = "postgresql://{}:{}@supersetdb:5432/{}".format(os.environ['POSTGRESQL_USERNAME'], os.environ['POSTGRESQL_PASSWORD'], os.environ['POSTGRESQL_DATABASE']) SQLALCHEMY_TRACK_MODIFICATIONS = True - SECRET_KEY = os.getenv('SUPERSET_SECRET_KEY','$(SUPERSET_SECRET_KEY)') + SECRET_KEY = os.getenv('SUPERSET_SECRET_KEY','') DATA_DIR = '/var/lib/superset' - LOG_LEVEL = '$(LOG_LEVEL)' + LOG_LEVEL = 'INFO' FEATURE_FLAGS = { 'ENABLE_TEMPLATE_PROCESSING': True, } diff --git a/superset/base/supersetdb-deployment.yaml b/superset/base/supersetdb-deployment.yaml index 83a197d63..8e9069369 100644 --- a/superset/base/supersetdb-deployment.yaml +++ b/superset/base/supersetdb-deployment.yaml @@ -20,24 +20,23 @@ spec: spec: containers: - name: postgresql - image: "quay.io/internaldatahub/postgresql-96-centos7:9.6" + image: supersetdb env: - name: POSTGRESQL_USER valueFrom: secretKeyRef: key: database-user - name: supersetdb + name: $(superset_db_secret) - name: POSTGRESQL_PASSWORD valueFrom: secretKeyRef: key: database-password - name: supersetdb + name: $(superset_db_secret) - name: POSTGRESQL_DATABASE valueFrom: secretKeyRef: key: database-name - name: supersetdb - + name: $(superset_db_secret) imagePullPolicy: Always livenessProbe: exec: @@ -56,11 +55,11 @@ spec: timeoutSeconds: 1 resources: requests: - cpu: $(SUPERSET_DB_CPU_REQUESTS) - memory: $(SUPERSET_DB_MEMORY_REQUESTS) + cpu: $(superset_db_cpu_requests) + memory: $(superset_db_memory_requests) limits: - cpu: $(SUPERSET_DB_CPU_LIMITS) - memory: $(SUPERSET_DB_MEMORY_LIMITS) + cpu: $(superset_db_cpu_limits) + memory: $(superset_db_memory_limits) volumeMounts: - mountPath: /var/lib/pgsql/data name: "supersetdb-data" diff --git a/superset/base/supersetdb-secret.yaml b/superset/base/supersetdb-secret.yaml index 3386034b4..d5b18264a 100644 --- a/superset/base/supersetdb-secret.yaml +++ b/superset/base/supersetdb-secret.yaml @@ -9,6 +9,6 @@ metadata: template.openshift.io/expose-username: '{.data[''database-user'']}' name: supersetdb stringData: - database-name: supersetdb - database-password: $(SUPERSET_DB_PASSWORD) - database-user: $(SUPERSET_DB_USERNAME) + database-name: superset + database-password: changeme + database-user: changeme diff --git a/tests/setup/kfctl_openshift.yaml b/tests/setup/kfctl_openshift.yaml index ccdc4dfde..b5ddfac16 100644 --- a/tests/setup/kfctl_openshift.yaml +++ b/tests/setup/kfctl_openshift.yaml @@ -18,7 +18,7 @@ spec: - kustomizeConfig: parameters: # Note: The admin username is admin - - name: SUPERSET_ADMIN_PASSWORD + - name: superset_admin_password value: admin repoRef: name: manifests