a novel mechanism to create mountpoint sub-directories in qiita's BASE_DIR

Images/nginx/nginx_qiita.conf

@@ -57,7 +57,7 @@ http {
 
           # CHANGE ME: This should match the WORKING_DIR in your qiita
           # config. E.g.,
-          alias /qiita/qiita_db/support_files/test_data/working_dir/;
+          alias /qiita_data/working_dir/;
         }
 
         # protected location
@@ -66,7 +66,7 @@ http {
 
             # CHANGE ME: This should match the BASE_DATA_DIR in your qiita
             # config. E.g.,
-            alias /qiita/qiita_db/support_files/test_data/;
+            alias /qiita_data/;
         }
 
         # enables communiction through websockets.

Images/plugin_collector/fix_test_db.py

@@ -9,14 +9,19 @@
 is_test = qiita_config['main']['TEST_ENVIRONMENT'].upper() == 'TRUE'
 print("qiita is in %s mode." % ('TEST' if is_test else 'PRODUCTIVE'))
 
-if is_test:
-    conn = psycopg2.connect(database=qiita_config['postgres']['DATABASE'],
-                            host=qiita_config['postgres']['HOST'],
-                            user=qiita_config['postgres']['ADMIN_USER'],
-                            password=qiita_config['postgres']['ADMIN_PASSWORD'],
-                            port=qiita_config['postgres']['PORT'])
-    cursor = conn.cursor()
+conn = psycopg2.connect(database=qiita_config['postgres']['DATABASE'],
+                        host=qiita_config['postgres']['HOST'],
+                        user=qiita_config['postgres']['ADMIN_USER'],
+                        password=qiita_config['postgres']['ADMIN_PASSWORD'],
+                        port=qiita_config['postgres']['PORT'])
+cursor = conn.cursor()
+
+# update conda env for qiita private plugins
+sql = "UPDATE qiita.software SET environment_script = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/qiita' WHERE description = 'Internal Qiita jobs';"
+cursor.execute(sql)
+conn.commit()
 
+if is_test:
     fps_plugin_configs = glob('/qiita_plugins/*.conf')
     print("Updating plugin credentials in dummy test DB with actual values from %i plugins." % len(fps_plugin_configs))
     for i, fp_plugin_config in enumerate(fps_plugin_configs):

Images/plugin_collector/startup_plugin_collector.sh

@@ -8,5 +8,7 @@ done
 # it seems to be necessary to give the plugin container some lead time
 # TODO: this might be more appropriately be addressed with healthchecks in the compose file
 sleep 3
+# create WORKING_DIR, UPLOAD_DATA_DIR and BASE_DATA_DIR in shared volume
+mkdir -p /qiita_data/working_dir/ /qiita_data/uploads/
 python3 /collect_configs.py
 python3 /fix_test_db.py

Images/qiita/config_portal.cfg

@@ -0,0 +1,43 @@
+# ---------- Base information for the website ----------
+[sitebase]
+# Logo should be 100px by 40px
+LOGO = /static/img/logo-clear.png
+# Full path to portal custom CSS styling file
+CSS_FP =
+TITLE = Qiita
+
+# ---------- Welcome text on index page ----------
+[index]
+HEADER = Qiita Spots Patterns
+TEXT = <p align="justify">
+  Qiita (<i>canonically pronounced cheetah</i>) is an entirely
+  <strong>open-source</strong> microbial study management platform. It allows
+  users to keep track of multiple studies with multiple ‘omics data.
+  Additionally, Qiita is capable of supporting multiple analytical pipelines
+  through a 3rd-party plugin system, allowing the user to have a single entry
+  point for all of their analyses.
+  </p>
+  <p align="justify">
+  Qiita provides database and compute resources to the global community,
+  alleviating the technical burdens that are typically limiting for
+  researchers studying microbial ecology (e.g. familiarity with the command
+  line or access to compute power).
+  </p>
+  <p align="justify">
+  Qiita’s platform allows for quick reanalysis of the datasets that have been
+  deposited using the latest analytical technologies. This means that Qiita’s
+  internal datasets are living data that is periodically re-annotated
+  according to current best practices.
+  </p>
+  <p align="justify">
+  For more information about how to use Qiita, visit the
+  <a href="static/doc/html/index.html">documentation</a>.
+  </p>
+  <p align="justify">
+  Note that you should be logged into the system to access any studies and
+  files available.
+  </p>
+
+# ---------- Study listing page ----------
+[study_list]
+EXAMPLE_SEARCH = env_matter = soil

Images/qiita/config_qiita_oidc.cfg

@@ -26,16 +26,16 @@ REQUIRE_APPROVAL = True
 BASE_URL = https://qiita-container-anna-nginx-1:8383
 
 # Download path files
-UPLOAD_DATA_DIR = /qiita/qiita_db/support_files/test_data/uploads/
+UPLOAD_DATA_DIR = /qiita_data/uploads/
 
 # Working directory path
-WORKING_DIR = /qiita/qiita_db/support_files/test_data/working_dir/
+WORKING_DIR = /qiita_data/working_dir/
 
 # Maximum upload size (in Gb)
 MAX_UPLOAD_SIZE = 100
 
 # Path to the base directory where the data files are going to be stored
-BASE_DATA_DIR = /qiita/qiita_db/support_files/test_data/
+BASE_DATA_DIR = /qiita_data/
 
 # Valid upload extension, comma separated. Empty for no uploads
 VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual
@@ -181,7 +181,7 @@ PORTAL = QIITA
 PORTAL_DIR =
 
 # Full path to portal styling config file
-PORTAL_FP =
+PORTAL_FP = /qiita_configurations/config_portal.cfg
 
 # The center latitude of the world map, shown on the Stats map.
 # Defaults to 40.01027 (Boulder, CO, USA)

Images/qiita/qiita.dockerfile

@@ -62,7 +62,8 @@ RUN pip install -e qiita --no-binary redbiom
 
 # Copy Bash Script to run Qiita to the container. start_qiita differentiates between one "master" and multiple workers
 COPY start_qiita.sh .
-RUN chmod 755 start_qiita.sh
+COPY start_qiita-initDB.sh .
+RUN chmod 755 start_qiita.sh start_qiita-initDB.sh
 
 RUN apt-get install -y curl
 COPY start_plugin.py /start_plugin.py
@@ -71,4 +72,7 @@ RUN chmod a+x /start_plugin.py
 # hide certificate and server configuration copy from source code
 RUN rm -rf /qiita/qiita_core/support_files
 
+# hide default configurations from github sources
+RUN rm -f /qiita/qiita_pet/nginx_example.conf /qiita/qiita_pet/supervisor_example.conf /qiita/qiita_pet/support_files/config_portal.cfg
+
 # CMD ["conda", "run", "-n", "qiita"]

Images/qiita/start_qiita-initDB.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+CONDA_DIR=/opt/conda
+ENV_NAME=qiita
+#PORT=21174
+
+# We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
+source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
+# To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
+grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err

Images/qiita/start_qiita.sh

@@ -10,10 +10,10 @@ if [ -n "${MASTER}" ] && [ ! -d /qiita/qiita_db/__pycache__ ]; then
 	source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; pip install -e . --no-binary redbiom;
 fi
 
-# We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
-source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
-# To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
-grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err
+# # We execute qiita-env make every time. We expect that it will fail always but the very first time, as the qiita DB should exist from then on
+# source $CONDA_DIR/etc/profile.d/conda.sh; conda activate $CONDA_DIR/envs/$ENV_NAME; cd /qiita; qiita-env make --no-load-ontologies 2> .env-make.err || true
+# # To avoid confusing the user, STDERR is written into a file and only reported if it does not contain the text that we expect to see if it just reports the existing DB
+# grep 'already present on the system. You can drop it by running' .env-make.err > /dev/null || cat .env-make.err
 
 # This was commented out bc it stopped working anymore and i was focusing on fixing something else, if you create the database for the first
 # time you will have to pick the appropriate options.

Makefile

@@ -84,7 +84,7 @@ plugin: Images/qtp-biom/trigger.py Certificates/
 	chmod a+rw ./logs/nginx_access.log ./logs/nginx_error.log
 	touch .built_image_nginx
 
-.built_image_qiita: Images/qiita/qiita.dockerfile Images/qiita/config_qiita_oidc.cfg Images/qiita/start_qiita.sh Images/qiita/supervisor_foreground.conf Images/qiita/start_plugin.py
+.built_image_qiita: Images/qiita/qiita.dockerfile Images/qiita/config_qiita_oidc.cfg Images/qiita/start_qiita.sh Images/qiita/start_qiita-initDB.sh Images/qiita/supervisor_foreground.conf Images/qiita/start_plugin.py Images/qiita/config_portal.cfg
 	test -d src/qiita || git clone -b auth_oidc https://github.com/jlab/qiita.git src/qiita
 	# remove configuration and certificate files from upstream qiita repo
 	rm -rf src/qiita/qiita_core/support_files

compose.yaml

@@ -18,6 +18,26 @@ services:
     ports:
       - "15432:5432"
 
+  qiita-initialize-db:
+    image: local-qiita:latest
+    command: ['/start_qiita-initDB.sh']
+    depends_on:
+      - qiita-db
+    env_file:
+      - './environments/qiita.env'
+    environment:
+      - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg
+      - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
+    volumes:
+      - ./src/qiita:/qiita:U
+      - qiita-data:/qiita_data
+      - server-plugin-configs:/qiita_plugins
+      - ./logs:/logs
+      - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r
+      - ./Images/qiita/config_portal.cfg:/qiita_configurations/config_portal.cfg:r
+    networks:
+      - qiita-net
+
   qiita:
       image: local-qiita:latest
       build: # image wird hier direkt gebaut
@@ -42,10 +62,11 @@ services:
         - MASTER=--master
         - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
       volumes:
-        #- qiita-data:/qiita
+        - qiita-data:/qiita_data
         - ./src/qiita:/qiita:U
         - ./logs:/logs
         - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r
+        - ./Images/qiita/config_portal.cfg:/qiita_configurations/config_portal.cfg:r
         - server-plugin-configs:/qiita_plugins
         - server-certificates:/qiita_certificates
         # - ./Images/qiita/start_qiita.sh:/qiita/start_qiita.sh
@@ -69,8 +90,10 @@ services:
         # - 127.0.0.1:8383:8383 #damit bur ich dran komme
       restart: no
       depends_on:
-        - redis
-        - plugin-collector
+        redis:
+          condition: service_started
+        plugin-collector:
+          condition: service_completed_successfully
       env_file:
         - './environments/qiita.env'
       environment:
@@ -80,9 +103,10 @@ services:
         - MASTER=
         - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
       volumes:
-        - qiita-data:/qiita
+        - qiita-data:/qiita_data
         - ./logs:/logs
         - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r
+        - ./Images/qiita/config_portal.cfg:/qiita_configurations/config_portal.cfg:r
         - server-plugin-configs:/qiita_plugins
         - server-certificates:/qiita_certificates
         - ./src/qiita:/qiita:U
@@ -154,7 +178,7 @@ services:
     depends_on:
       - qiita
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       #- ./logs/nginx_access.log:/logs/nginx_access.log
       #- ./logs/nginx_error.log:/logs/nginx_error.log
       - ./logs:/logs
@@ -171,15 +195,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-biom/lib/python3.8/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qtp-biom/lib/python3.8/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -191,15 +213,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-sequencing/lib/python3.9/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qtp-sequencing/lib/python3.9/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -211,15 +231,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qp-target-gene/lib/python2.7/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qp-target-gene/lib/python2.7/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -231,15 +249,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-visualization/lib/python3.6/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qtp-visualization/lib/python3.6/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -251,15 +267,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -271,15 +285,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/deblur/lib/python3.5/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/deblur/lib/python3.5/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -291,15 +303,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qiime2/lib/python3.8/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -311,15 +321,13 @@ services:
     # tty: true
     restart: no
     volumes:
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
     environment:
       # TODO: is there a more elegant way to obtain this path?
       - REQUESTS_CA_BUNDLE=/opt/conda/envs/qtp-job-output-folder/lib/python3.6/site-packages/certifi/cacert.pem
       - SSL_CERT_FILE=/opt/conda/envs/qtp-job-output-folder/lib/python3.6/site-packages/certifi/cacert.pem
       - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
-    #depends_on:
-    #- qiita
     networks:
       - qiita-net
 
@@ -333,20 +341,29 @@ services:
     volumes:
       - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r
       - server-plugin-configs:/qiita_plugins
-      - qiita-data:/qiita
+      - qiita-data:/qiita_data
       - server-certificates:/qiita_certificates
       #- /Daten/Git/jlab/qiita-container-anna/Images/plugin_collector/collect_configs.py:/collect.py
       #- /Daten/Git/jlab/qiita-container-anna/Images/plugin_collector/fix_test_db.py:/fix_test_db.py
     depends_on:
-      - qtp-biom  # one of the plugins
-      - qtp-sequencing
-      - qp-target-gene
-      - qtp-visualization
-      - qtp-diversity
-      - qp-deblur
-      - qp-qiime2
-      - qtp-job-output-folder
-      - qiita-db  # as values in qiita DB might be updated
+      qiita-initialize-db:
+        condition: service_completed_successfully
+      qtp-biom:  # one of the plugins
+        condition: service_started
+      qtp-sequencing:
+        condition: service_started
+      qp-target-gene:
+        condition: service_started
+      qtp-visualization:
+        condition: service_started
+      qtp-diversity:
+        condition: service_started
+      qp-deblur:
+        condition: service_started
+      qp-qiime2:
+        condition: service_started
+      qtp-job-output-folder:
+        condition: service_started
     environment:
       - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:"
     command: ['/startup_plugin_collector.sh']