adding Tobi's multiqc

Images/qp-multiqc/qp-multiqc.dockerfile

@@ -0,0 +1,72 @@
+FROM ubuntu:24.04
+
+ARG MINIFORGE_VERSION=24.1.2-0
+
+ENV CONDA_DIR=/opt/conda
+ENV PATH=${CONDA_DIR}/bin:${PATH}
+
+RUN apt-get -y update
+RUN apt-get -y --fix-missing install \
+	git \
+	wget \
+	libpq-dev \
+	python3-dev \
+	gcc \
+	build-essential
+
+# install miniforge3 for "conda"
+# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile
+RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \
+	/bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \
+	echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
+	echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc && \
+	conda init && \
+	rm -f /tmp/miniforge3.sh
+
+# install tornado based trigger layer in base environment
+RUN pip install -U pip
+RUN conda install tornado
+COPY trigger.py /trigger.py
+
+# Create conda env
+RUN conda create --quiet -n multiqc -c bioconda python=3.9 pip fastqc multiqc
+# Make RUN commands use the new environment:
+# append --format docker to the build command, see https://github.com/containers/podman/issues/8477
+SHELL ["conda", "run", "-p", "/opt/conda/envs/multiqc", "/bin/bash", "-c"]
+
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN pip install -U pip
+
+RUN pip install https://github.com/qiita-spots/qiita_client/archive/master.zip
+RUN pip install https://github.com/qiita-spots/qiita-files/archive/master.zip
+RUN git clone https://github.com/jlab/qp-multiqc.git
+WORKDIR qp-multiqc
+RUN pip install -e .
+RUN pip install pip-system-certs
+
+# TODO: should the plugin get the server configuration?!
+RUN export QIITA_CONFIG_FP=/qiita/config_qiita_oidc.cfg
+
+WORKDIR /
+
+COPY start_qp-multiqc.sh .
+RUN chmod 755 start_qp-multiqc.sh
+
+RUN mkdir -p /unshared_plugins
+ENV QIITA_PLUGINS_DIR=/unshared_plugins/
+
+##  Export cert and config filepaths
+COPY Certificates /unshared_certificates
+RUN cat /unshared_certificates/stefan_rootca.crt >> `python -c "import certifi; print(certifi.where())"`  # append own rootCA onto chain of trust
+RUN export REQUESTS_CA_BUNDLE=`python -c "import certifi; print(certifi.where())"`
+RUN export SSL_CERT_FILE=`python -c "import certifi; print(certifi.where())"`
+
+#RUN export QIITA_ROOTCA_CERT=/unshared_certificates/ci_rootca.crt
+#RUN sed -i "s/f'Entered BaseQiitaPlugin._register_command({command.name})'/'Entered BaseQiitaPlugin._register_command(%s)' % command.name/"  $CONDA_PREFIX/lib/python3.5/site-packages/qiita_client/plugin.py
+RUN chmod u+x /qp-multiqc/scripts/configure_qp_multiqc
+RUN /qp-multiqc/scripts/configure_qp_multiqc --env-script "true" --server-cert /unshared_certificates/stefan_server.crt
+RUN sed -i -E "s/^START_SCRIPT = .+/START_SCRIPT = python \/start_plugin.py qp-multiqc/" /unshared_plugins/*.conf
+
+CMD ["./start_qp-multiqc.sh"]

Images/qp-multiqc/start_qp-multiqc.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cd / && python trigger.py multiqc start_qp_multiqc /qp-multiqc
+
+tail -f /dev/null

Makefile

@@ -31,52 +31,52 @@ Certificates/: Images/plugin_collector/stefan_csr.conf Images/plugin_collector/s
 plugin: Images/qtp-biom/trigger.py Certificates/
 	cp -r $^ $(tmpdir)/
 
-.built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile Images/qtp-biom/start_qtp-biom.sh
+.built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile Images/qtp-biom/start_qtp-biom.sh Certificates/*
 	test -d src/qtp-biom || git clone https://github.com/qiita-spots/qtp-biom.git src/qtp-biom
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qtp-sequencing: Images/qtp-sequencing/qtp-sequencing.dockerfile Images/qtp-sequencing/start_qtp-sequencing.sh
+.built_image_qtp-sequencing: Images/qtp-sequencing/qtp-sequencing.dockerfile Images/qtp-sequencing/start_qtp-sequencing.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh
+.built_image_qp-target-gene: Images/qp-target-gene/qp-target-gene.dockerfile Images/qp-target-gene/start_qp-target-gene.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qtp-visualization: Images/qtp-visualization/qtp-visualization.dockerfile Images/qtp-visualization/start_qtp-visualization.sh
+.built_image_qtp-visualization: Images/qtp-visualization/qtp-visualization.dockerfile Images/qtp-visualization/start_qtp-visualization.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qtp-diversity: Images/qtp-diversity/qtp-diversity.dockerfile Images/qtp-diversity/start_qtp-diversity.sh
+.built_image_qtp-diversity: Images/qtp-diversity/qtp-diversity.dockerfile Images/qtp-diversity/start_qtp-diversity.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qp-deblur: Images/qp-deblur/qp-deblur.dockerfile Images/qp-deblur/start_qp-deblur.sh
+.built_image_qp-deblur: Images/qp-deblur/qp-deblur.dockerfile Images/qp-deblur/start_qp-deblur.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qp-qiime2: Images/qp-qiime2/qp-qiime2.dockerfile Images/qp-qiime2/start_qp-qiime2.sh
+.built_image_qp-qiime2: Images/qp-qiime2/qp-qiime2.dockerfile Images/qp-qiime2/start_qp-qiime2.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
-.built_image_qtp-job-output-folder: Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile Images/qtp-job-output-folder/start_qtp-job-output-folder.sh
+.built_image_qtp-job-output-folder: Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile Images/qtp-job-output-folder/start_qtp-job-output-folder.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
-	cp $^ $(TMPDIR)
+	cp -r $^ $(TMPDIR)
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
 	touch .built_image_`basename $< | cut -d "." -f 1`
 
@@ -94,14 +94,20 @@ plugin: Images/qtp-biom/trigger.py Certificates/
 	cd Images/qiita && $(PODMAN_BIN) build . -f `basename $<` $(PODMAN_FLAGS) -t local-qiita
 	touch .built_image_qiita
 
-.built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh
+.built_image_qp-multiqc: Images/qp-multiqc/qp-multiqc.dockerfile Images/qp-multiqc/start_qp-multiqc.sh Certificates/*
+	tmpdir=$(TMPDIR) $(MAKE) plugin
+	cp -r $^ $(TMPDIR)
+	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-`basename $< | cut -d "." -f 1`
+	touch .built_image_`basename $< | cut -d "." -f 1`
+
+.built_image_plugin_collector: Images/plugin_collector/plugin_collector.dockerfile Images/plugin_collector/fix_test_db.py Images/plugin_collector/collect_configs.py Images/plugin_collector/startup_plugin_collector.sh Certificates/*
 	tmpdir=$(TMPDIR) $(MAKE) plugin
 	cp $^ $(TMPDIR)
 	cp -r Certificates/ $(tmpdir)/
 	$(PODMAN_BIN) build $(TMPDIR)/ -f $(TMPDIR)/`basename $<` $(PODMAN_FLAGS) -t local-plugin_collector
 	touch .built_image_plugin_collector
 
-images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder
+images: .built_image_qtp-biom .built_image_nginx .built_image_qiita .built_image_plugin_collector .built_image_qtp-sequencing .built_image_qp-target-gene .built_image_qtp-visualization .built_image_qtp-diversity .built_image_qp-deblur .built_image_qp-qiime2 .built_image_qp-qiime2 .built_image_qtp-job-output-folder .built_image_qp-multiqc
 
 environments/qiita_db.env: environments/qiita_db.env.example
 	cp environments/qiita_db.env.example environments/qiita_db.env

compose.yaml

@@ -343,6 +343,25 @@ services:
     networks:
       - qiita-net
 
+  qp-multiqc:
+    image: local-qp-multiqc:latest
+    command: ['./start_qp-multiqc.sh']
+    # network_mode: host
+    # stdin_open: true
+    # tty: true
+    restart: no
+    volumes:
+      - qiita-data:/qiita_data
+      - ./Images/qiita/config_qiita_oidc.cfg:/qiita_configurations/qiita_server.cfg:r  # TODO: do we really want to expose server settings to the plugin?
+    environment:
+      # TODO: is there a more elegant way to obtain this path?
+      - REQUESTS_CA_BUNDLE=/opt/conda/envs/multiqc/lib/python3.9/site-packages/certifi/cacert.pem
+      - SSL_CERT_FILE=/opt/conda/envs/multiqc/lib/python3.9/site-packages/certifi/cacert.pem
+      - QIITA_CLIENT_DEBUG_LEVEL=DEBUG
+      - QIITA_CONFIG_FP=/qiita_configurations/qiita_server.cfg
+    networks:
+      - qiita-net
+
   plugin-collector:
     # prior to qiita (master and worker) start up, iterates through the QIITA_PLUGINS : separated list of plugin containers
     # to compile all q*.conf files from plugin containers in the server-plugin-configs volume
@@ -374,8 +393,10 @@ services:
         condition: service_started
       qtp-job-output-folder:
         condition: service_started
+      qp-multiqc:
+        condition: service_started
     environment:
-      - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:"
+      - QIITA_PLUGINS="qtp-biom:qtp-sequencing:qp-target-gene:qtp-visualization:qtp-diversity:qp-deblur:qp-qiime2:qtp-job-output-folder:qp-multiqc:"
     command: ['/startup_plugin_collector.sh']
 
 networks: