Merge branch 'staging' into mongo-backup

mistio · Oct 26, 2018 · daf8d5a · daf8d5a
2 parents abd4828 + 0cf3ddb
commit daf8d5a
Show file tree

Hide file tree

Showing 35 changed files with 599 additions and 128 deletions.
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,5 @@ bower_components
 .cache
 pip-selfcheck.json
 keys
+/chart/charts
+/chart/requirements.lock
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -226,7 +226,7 @@ compose:
 
 .deploy_template: &deploy_template
   stage: deploy
-  image: dtzar/helm-kubectl
+  image: dtzar/helm-kubectl:2.10.0
   before_script:
     - |
         set -e
@@ -293,7 +293,12 @@ compose:
             rm -rf $tls_tmpdir
           fi
 
+          helm init --client-only
+          helm repo update
+          helm dependency update ./chart/
+
           helm upgrade --install --namespace $namespace \
+            --wait --timeout=600 \
             -f values.yaml \
             --set image.tag="$CI_COMMIT_SHA" \
             --set image.registry="$REGISTRY_IMAGE" \
@@ -317,7 +322,7 @@ compose:
           while kubectl --namespace $namespace get pods | grep -v -e NAME -e Running -e Completed -e Evicted -e OutOfcpu; do
             sleep 10 && echo '.'
           done
-          while !(curl -sSLf "$CI_ENVIRONMENT_URL"); do
+          while !(curl -sSLf "$CI_ENVIRONMENT_URL" >/dev/null); do
             sleep 5 && echo 'Environment not ready yet...'
           done
         }
@@ -338,6 +343,9 @@ deploy test:
       enableDevUsers: true
       nodeSelector:
         cloud.google.com/gke-nodepool: generic
+      rabbitmq:
+        nodeSelector:
+          cloud.google.com/gke-nodepool: generic
 
 .deploy_review_template: &deploy_review_template
   <<: *deploy_template
@@ -353,6 +361,9 @@ deploy test:
         password: $SENDGRID_PASSWORD
       nodeSelector:
         cloud.google.com/gke-nodepool: generic
+      rabbitmq:
+        nodeSelector:
+          cloud.google.com/gke-nodepool: generic
 
 deploy review:
   <<: *deploy_review_template
@@ -412,6 +423,7 @@ API:main:schedules: *api_test_template
 API:main:scripts: *api_test_template
 API:main:rbac: *api_test_template
 API:main:ip_whitelisting: *api_test_template
+API:main:volumes: *api_test_template
 
 .API:plugin:orchestration: *api_test_template
 .API:plugin:tunnels: *api_test_template
@@ -476,7 +488,7 @@ UI:ip-whitelisting:
 
 .undeploy_template: &undeploy_template
   stage: undeploy
-  image: dtzar/helm-kubectl
+  image: dtzar/helm-kubectl:2.10.0
   variables:
     GIT_STRATEGY: none
   before_script:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,46 @@
 # Changelog
 
+## v3.3.0 (23 October 2018)
+
+* Feature: Support for block storage volumes in GCE, EC2, OpenStack, DigitalOcean
+* Feature: Automatic db migrations
+* Feature: Display org logo in user menu
+* Feature: Resize GCE machines
+* Feature: Allow to create multiple interfaces and assign static IPs to KVM guest VMs
+* Feature: Implement VM cloning for KVM
+* Feature: Support snapshots in vSphere
+* Feature: Allow to enable/disable and edit the window/frequency of a NoDataRule
+* Feature: Saved searches in log listings
+* Feature: Cloud listing page
+* Change: Use html5 date & time inputs
+* Change: Improve performance when applying patches to the model over websocket
+* Change: Reduce poller update threshold to 90 secs
+* Change: Add k8s deployment specific script that displays online portal users
+* Bugfix: Properly pass params when running scheduled scripts
+* Bugfix: Display prices in resize dialog correctly for DigitalOcean
+
+## v3.2.0 (6 Aug 2018)
+
+Performance optimizations, preliminary support for Alibaba Cloud (Aliyun ECS), minor bugfixes
+
+* Feature: Initial support for Alibaba Cloud
+* Feature: Configure default post deploy steps for selected clouds
+* Feature: Support VNC console action on OpenStack
+* Bugfix: Fix influxdb mountpath in helm chart
+* Bugfix: Fix VCloud OS detection
+* Bugfix: Fix vSphere machine listing issue
+* Bugfix: Fix load graph for many machines
+* Change: Expose more settings for rabbitmq, gocky, cilia in helm chart
+* Change: Upgrade gocky images
+* Change: Configurable pollers in helm chart
+* Change: Add flag to disable machine polling acceleration
+* Change: Optimize tag query when calculating machine cost
+* Change: Re-implement metering for checks and datapoints based on rate/sec
+* Change: Dont probe stopped machines or localhost
+* Change: Dont run update_poller task if not necessary
+* Change: Import middleware chains from plugins, if defined
+* Change: Make scheduler configurable in bin/poller
+
 
 ## v3.1.0 (16 Jul 2018)
 

diff --git a/api b/api
diff --git a/chart/requirements.yaml b/chart/requirements.yaml
@@ -0,0 +1,6 @@
+dependencies:
+  - alias: rabbitmq
+    condition: rabbitmq.enabled
+    repository: https://kubernetes-charts.storage.googleapis.com/
+    name: rabbitmq-ha
+    version: 1.10.0
diff --git a/chart/templates/apply-migrations-job.yaml b/chart/templates/apply-migrations-job.yaml
@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: apply-migrations
+  annotations:
+    # This is what defines this resource as a hook. Without this line, the
+    # job is considered part of the release.
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": before-hook-creation
+spec:
+  backoffLimit: 10
+  template:
+    spec:
+      imagePullSecrets:
+{{ toYaml .Values.image.secrets | indent 8 }}
+      containers:
+      - args:
+        - bin/migrate
+        image: {{ .Values.image.registry }}/api:{{ .Values.image.tag }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        name: apply-migrations
+        resources: {}
+        volumeMounts:
+        - name: {{ .Values.productShortName }}-config
+          mountPath: /etc/{{ .Values.productShortName }}/settings/settings.py
+          subPath: settings.py
+      volumes:
+      - name: {{ .Values.productShortName }}-config
+        configMap:
+          name: {{ .Values.productShortName }}
+      restartPolicy: Never
diff --git a/chart/templates/beat-deployment.yaml b/chart/templates/beat-deployment.yaml
@@ -29,7 +29,7 @@ spec:
             memory: 250Mi
             cpu: 200m
           requests:
-            memory: 125Mi
+            memory: 150Mi
             cpu: 50m
         volumeMounts:
         - name: {{ .Values.productShortName }}-config

diff --git a/chart/templates/celery-deployment.yaml b/chart/templates/celery-deployment.yaml
@@ -1,30 +1,29 @@
-{{ $values := .Values -}}
-{{ range $name, $settings := $values.deployment.celery -}}
+{{ range $name, $settings := $.Values.deployment.celery -}}
 {{ if $settings.enabled -}}
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
   labels:
-    {{ $values.productShortName }}.service: celery-{{ $name }}
+    {{ $.Values.productShortName }}.service: celery-{{ $name }}
   name: celery-{{ $name }}
 spec:
   replicas: {{ $settings.replicas | default "1" }}
   template:
     metadata:
       labels:
-        {{ $values.productShortName }}.service: celery-{{ $name }}
+        {{ $.Values.productShortName }}.service: celery-{{ $name }}
       annotations:
         # Trigger modification of deployment & restart of pods if config changes.
         checksum/config: {{ include (print $.Template.BasePath "/main-configmap.yaml") $ | sha256sum }}
     spec:
       imagePullSecrets:
-{{ toYaml $values.image.secrets | indent 8 }}
+{{ toYaml $.Values.image.secrets | indent 8 }}
       containers:
       - name: celery-{{ $name }}
         args:
         - bin/celery-{{ $settings.pool | default "prefork" }}
-        image: {{ $values.image.registry }}/api:{{ $values.image.tag }}
-        imagePullPolicy: {{ $values.image.pullPolicy }}
+        image: {{ $.Values.image.registry }}/api:{{ $.Values.image.tag }}
+        imagePullPolicy: {{ $.Values.image.pullPolicy }}
         env:
         - name: QUEUES
           value: "{{ range $i, $q := $settings.queues }}{{ if gt $i 0 }},{{ end }}{{ $q }}{{ end }}"
@@ -36,16 +35,28 @@ spec:
             memory: {{ $settings.resources.limits.memory | default "1000Mi" }}
             cpu: {{ $settings.resources.limits.cpu | default "1000m" }}
         volumeMounts:
-        - name: {{ $values.productShortName }}-config
-          mountPath: /etc/{{ $values.productShortName }}/settings/settings.py
+        - name: {{ $.Values.productShortName }}-config
+          mountPath: /etc/{{ $.Values.productShortName }}/settings/settings.py
           subPath: settings.py
+{{- if $settings.preemptible }}
+{{- if $.Values.preemptible.tolerations }}
+      tolerations:
+{{ toYaml $.Values.preemptible.tolerations | indent 8 }}
+{{- end }}
+{{- if $.Values.preemptible.nodeAffinity }}
+      affinity:
+        nodeAffinity:
+{{ toYaml $.Values.preemptible.nodeAffinity | indent 10 }}
+{{- end }}
+{{- end }}
+      terminationGracePeriodSeconds: {{ $settings.gracePeriod | default 300 }}
       volumes:
-      - name: {{ $values.productShortName }}-config
+      - name: {{ $.Values.productShortName }}-config
         configMap:
-          name: {{ $values.productShortName }}
-{{- if $values.nodeSelector }}
+          name: {{ $.Values.productShortName }}
+{{- if $.Values.nodeSelector }}
       nodeSelector:
-{{ toYaml $values.nodeSelector | indent 8 }}
+{{ toYaml $.Values.nodeSelector | indent 8 }}
 {{- end }}
 ---
 {{ end }}

diff --git a/chart/templates/celery-exporter-deployment.yaml b/chart/templates/celery-exporter-deployment.yaml
@@ -1,3 +1,4 @@
+{{ if .Values.prometheus.enabled -}}
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
@@ -19,7 +20,11 @@ spec:
       - name: celery-exporter
         args:
         - --broker
-        - amqp://guest:guest@rabbitmq/
+        {{ if .Values.rabbitmq.enabled -}}
+        - "amqp://{{ .Values.rabbitmq.rabbitmqUsername }}:{{ .Values.rabbitmq.rabbitmqPassword }}@{{ .Release.Name }}-rabbitmq:5672/"
+        {{- else -}}
+        - "amqp://{{ .Values.rabbitmqExternal.username }}:{{ .Values.rabbitmqExternal.password }}@{{ .Values.rabbitmqExternal.host }}:{{ .Values.rabbitmqExternal.port }}/"
+        {{- end }}
         - --enable-events
         image: dimrozakis/celery-exporter
         imagePullPolicy: {{ .Values.image.pullPolicy }}
@@ -34,3 +39,4 @@ spec:
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}
 {{- end }}
+{{- end }}
diff --git a/chart/templates/celery-exporter-service.yaml b/chart/templates/celery-exporter-service.yaml
@@ -0,0 +1,18 @@
+{{ if .Values.prometheus.enabled -}}
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{ .Values.productShortName }}.service: celery-exporter
+  annotations:
+    prometheus.io/path: /
+    prometheus.io/port: "8888"
+    prometheus.io/scrape: "true"
+  name: celery-exporter
+spec:
+  ports:
+  - name: exporter
+    port: 8888
+  selector:
+    {{ .Values.productShortName }}.service: celery-exporter
+{{- end }}
diff --git a/chart/templates/celery-exporter-servicemonitor.yaml b/chart/templates/celery-exporter-servicemonitor.yaml
@@ -0,0 +1,21 @@
+{{ if and .Values.prometheus.enabled .Values.prometheus.operator.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-celery
+  namespace: {{ .Values.prometheus.operator.serviceMonitor.namespace }}
+  labels:
+{{- if .Values.prometheus.operator.serviceMonitor.selector }}
+{{ toYaml .Values.prometheus.operator.serviceMonitor.selector | indent 4 }}
+{{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{ .Values.productShortName }}.service: celery-exporter
+  endpoints:
+  - port: exporter
+    interval: {{ .Values.prometheus.operator.serviceMonitor.interval }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+{{ end }}
diff --git a/chart/templates/celery-horizontalpodautoscaler.yaml b/chart/templates/celery-horizontalpodautoscaler.yaml
@@ -0,0 +1,42 @@
+{{ $values := .Values -}}
+{{ range $name, $settings := $values.deployment.celery -}}
+{{ if and $settings.enabled $settings.autoscaling.enabled -}}
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  labels:
+    {{ $values.productShortName }}.service: celery-{{ $name }}
+  name: celery-{{ $name }}
+spec:
+  scaleTargetRef:
+    apiVersion: extensions/v1beta1
+    kind: Deployment
+    name: celery-{{ $name }}
+  minReplicas: {{ $settings.autoscaling.minReplicas }}
+  maxReplicas: {{ $settings.autoscaling.maxReplicas }}
+  metrics:
+{{- if $settings.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: {{ $settings.autoscaling.targetCPUUtilizationPercentage }}
+{{- end }}
+{{- if $settings.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        targetAverageUtilization: {{ $settings.autoscaling.targetMemoryUtilizationPercentage }}
+{{- end }}
+{{- if and $.Values.prometheus.enabled $.Values.prometheus.operator.enabled }}
+    - type: Object
+      object:
+        target:
+          apiVersion: extensions/v1beta1
+          kind: Deployment
+          name: celery-{{ $name }}
+        metricName: hpa_custom_metrics_deployment_queues_length
+        targetValue: 10
+{{- end }}
+---
+{{ end }}
+{{- end }}
diff --git a/chart/templates/common-horizontalpodautoscaler.yaml b/chart/templates/common-horizontalpodautoscaler.yaml
@@ -0,0 +1,34 @@
+{{ $values := .Values -}}
+{{ range $name, $settings := $values.deployment -}}
+{{ if and (ne $name "celery") (ne $name "poller") ($settings.autoscaling) -}}
+{{ if default false $settings.autoscaling.enabled -}}
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  labels:
+    {{ $values.productShortName }}.service: {{ $name }}
+  name: {{ $name }}
+spec:
+  scaleTargetRef:
+    apiVersion: extensions/v1beta1
+    kind: Deployment
+    name: {{ $name }}
+  minReplicas: {{ $settings.autoscaling.minReplicas }}
+  maxReplicas: {{ $settings.autoscaling.maxReplicas }}
+  metrics:
+{{- if $settings.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: {{ $settings.autoscaling.targetCPUUtilizationPercentage }}
+{{- end }}
+{{- if $settings.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        targetAverageUtilization: {{ $settings.autoscaling.targetMemoryUtilizationPercentage }}
+{{- end }}
+---
+{{ end }}
+{{- end }}
+{{- end }}
diff --git a/chart/templates/landing-deployment.yaml b/chart/templates/landing-deployment.yaml
@@ -27,6 +27,11 @@ spec:
           limits:
             memory: {{ .Values.deployment.landing.resources.limits.memory | default "50Mi" }}
             cpu: {{ .Values.deployment.landing.resources.limits.cpu | default "200m" }}
+        lifecycle:
+          preStop:
+            exec:
+              # SIGTERM triggers a quick exit; gracefully terminate instead
+              command: ["/usr/sbin/nginx","-s","quit"]
 {{- if .Values.nodeSelector }}
       nodeSelector:
 {{ toYaml .Values.nodeSelector | indent 8 }}