elasticdump for elasticsearch data backup

Author: Jancis

charts/drupal/templates/_helpers.tpl

@@ -369,9 +369,18 @@ done
     {{- end }}
   {{- end }}
 
-  {{ if .Values.elasticsearch.enabled }}
-    {{ include "drupal.wait-for-elasticsearch-command" . }}
-  {{ end }}
+  {{- if .Values.elasticsearch.enabled }}
+    {{- include "drupal.wait-for-elasticsearch-command" . }}
+    {{- if and .Release.IsInstall .Values.referenceData.elasticsearch }}
+      if [[ -f /app/reference-data/elasticsearch.tar.gz || -f /app/reference-data/elasticsearch.tar.gz ]]; then
+        echo "Importing elasticsearch reference data"
+        mkdir /tmp/elasticsearch
+        tar -xzf /app/reference-data/elasticsearch.tar.gz -C /tmp/elasticsearch
+        multielasticdump --direction=load --input=/tmp/elasticsearch --output=http://${ELASTICSEARCH_HOST}:9200/
+        echo "Elasticsearch reference data imported"
+      fi
+    {{- end }}
+  {{- end }}
 
   {{ if .Release.IsInstall }}
     {{ .Values.php.postinstall.command }}
@@ -395,6 +404,7 @@ done
 {{- define "drupal.extract-reference-data" -}}
 set -e
 if [[ "$(drush status --fields=bootstrap)" = *'Successful'* ]] ; then
+
   echo "Dump reference database."
   dump_dir=/tmp/reference-data-export/
   mkdir "${dump_dir}"
@@ -462,6 +472,15 @@ if [[ "$(drush status --fields=bootstrap)" = *'Successful'* ]] ; then
     /app/reference-data/{{ $index }}
   {{ end -}}
   {{- end }}
+
+  {{- if .Values.referenceData.elasticsearch }}
+  echo "Creating reference data for elasticsearch"
+  mkdir /tmp/elasticsearch
+  multielasticdump --direction=dump --input=http://${ELASTICSEARCH_HOST}:9200/ --output=/tmp/elasticsearch
+  tar -czf /tmp/elasticsearch.tar.gz -C /tmp/elasticsearch .
+  cp /tmp/elasticsearch.tar.gz /app/reference-data
+  {{- end }}
+
 else
   echo "Drupal is not installed, skipping reference database dump."
 fi
@@ -519,6 +538,9 @@ fi
 
 {{- define "drupal.backup-command" -}}
   {{ include "drupal.backup-command.dump-database" . }}
+  {{- if and .Values.elasticsearch.enabled .Values.backup.elasticsearch }}
+  {{- include "drupal.backup-command.dump-elasticsearch" . }}
+  {{- end }}
   {{ include "drupal.backup-command.archive-store-backup" . }}
 {{- end }}
 
@@ -545,6 +567,24 @@ fi
   echo "Database backup complete."
 {{- end }}
 
+{{- define "drupal.backup-command.dump-elasticsearch" }}
+  set -e
+  
+  if ! command -v multielasticdump &> /dev/null
+  then
+    echo "multielasticdump could not be found. Elasticsearch backup failed."
+    exit 1
+  fi
+  
+  # Dump elasticsearch data
+  echo "Starting Elasticsearch backup."
+  mkdir /tmp/elasticsearch
+  multielasticdump --direction=dump --input=http://${ELASTICSEARCH_HOST}:9200/ --output=/tmp/elasticsearch
+  tar -czf /tmp/elasticsearch.tar.gz -C /tmp/elasticsearch .
+  
+  echo "Elasticsearch backup complete."
+{{- end }}
+
 {{- define "drupal.backup-command.archive-store-backup" -}}
 
   # Compress the database dump and copy it into the backup folder.
@@ -554,6 +594,13 @@ fi
 
   # Create a folder for the backup
   mkdir -p $BACKUP_LOCATION
+  
+  {{- if .Values.elasticsearch.enabled }}
+  # Copy the elasticsearch dump into the backup folder.
+  cp /tmp/elasticsearch.tar.gz $BACKUP_LOCATION/
+  {{- end }}
+
+  # Copy the database dump into the backup folder.
   cp /tmp/db.sql.gz $BACKUP_LOCATION/db.sql.gz
 
   {{- if not .Values.backup.skipFiles }}

charts/drupal/values.schema.json

@@ -448,6 +448,7 @@
         "storageClassName": { "type": "string" },
         "csiDriverName": { "type": "string" },
         "skipMount": { "type": "boolean" },
+        "elasticsearch": { "type": "boolean" },
         "resources": {
           "type": "object",
           "additionalProperties": false,
@@ -488,6 +489,7 @@
         "csiDriverName": { "type": "string" },
         "ignoreTableContent": { "type": "string" },
         "skipFiles": { "type": "boolean" },
+        "elasticsearch": { "type": "boolean" },
         "resources": {
           "type": "object",
           "additionalProperties": false,

charts/drupal/values.yaml

@@ -450,6 +450,8 @@ referenceData:
     - languages
     - php
 
+  elasticsearch: true
+
   # Files larger than this will not be included in reference data.
   maxFileSize: '5M'
 
@@ -530,8 +532,10 @@ backup:
   # These tables will have their content ignored from the backups.
   ignoreTableContent: '(cache|cache_.*)'
 
-  #  Do not backup files
-  #skipFiles: true
+  # Do not backup files
+  skipFiles: false
+
+  elasticsearch: true
 
   # Resources for the backup cron job.
   resources:

silta/php.Dockerfile

@@ -3,4 +3,9 @@ FROM wunderio/silta-php-fpm:8.2-fpm-v1
 
 COPY --chown=www-data:www-data . /app
 
+#####
+# Install Elasticdump
+RUN apk add npm nodejs && npm install -g elasticdump
+#####
+
 USER www-data

silta/shell.Dockerfile

@@ -1,4 +1,9 @@
 # Dockerfile for the Drupal container.
 FROM wunderio/silta-php-shell:php8.2-v1
 
+#####
+# Install Elasticdump (skip when php base image contains it)
+RUN apk add npm nodejs && npm install -g elasticdump
+#####
+
 COPY --chown=www-data:www-data . /app

silta/silta.yml

@@ -55,3 +55,16 @@ php:
       # The ~ symbol will be replaced by a random digit from 0 to 9.
       # This will avoid running all cron jobs at the same time.
       schedule: '~ 0 31 2 *'
+
+backup:
+  enabled: true
+  schedule: "*/10 * * * *"
+  retention: 3
+
+referenceData:
+  enabled: true
+  referenceEnvironment: 'feature/esdump'
+
+# TODO: wait for https://github.com/wunderio/silta-images/pull/180
+# TODO: remove elasticdump installation from local dockerfiles
+# TODO: remove test values