Merge branch 'qiita-spots:dev' into dev

Author: sjanssen2

.github/workflows/qiita-ci.yml

@@ -2,7 +2,7 @@
 
 on:
   push:
-    branches: [ dev ]
+    branches: [dev]
   pull_request:
 
 jobs:
@@ -45,7 +45,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: '3.9'
+          python-version: "3.9"
 
       - name: Basic dependencies install
         env:
@@ -148,6 +148,7 @@ jobs:
 
           echo "4. Setting up nginx"
           mkdir -p /usr/share/miniconda/envs/qiita/var/run/nginx/
+          sed -i "s|alias /Users/username|alias /home/runner/work/qiita|" ${PWD}/qiita_pet/nginx_example.conf
           nginx -c ${PWD}/qiita_pet/nginx_example.conf
 
           echo "5. Setting up qiita"
@@ -169,17 +170,20 @@ jobs:
           conda deactivate
 
           echo "8. Setting up SSH"
-          ssh-keygen -t rsa -b 4096 -N '' -f $PWD/qiita_ware/test/test_data/test_key
+          ssh-keygen -t ed25519 -a 200 -N '' -f $PWD/qiita_ware/test/test_data/test_key
           mkdir ~/.ssh/
           cp $PWD/qiita_ware/test/test_data/test_key* ~/.ssh/
           cat ~/.ssh/test_key.pub > ~/.ssh/authorized_keys
+          cat ~/.ssh/test_key.pub > ~/.ssh/authorized_keys2
           chmod 600 $PWD/qiita_ware/test/test_data/test_key*
           chmod 600 ~/.ssh/*
           chmod 700 ~/.ssh/
           echo "Connecting as $USER@localhost"
           # this line (and the -o StrictHostKeyChecking=no) is so the server
           # is added to the list of known servers
-          scp -O -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
+
+          # 05/22/25: commenting this line out as github actions is not allowing this step
+          # scp -O -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
 
       - name: Main tests
         shell: bash -l {0}
@@ -239,24 +243,24 @@ jobs:
     needs: main
     runs-on: ubuntu-latest
     steps:
-    - name: Coveralls Finished
-      uses: AndreMiras/coveralls-python-action@develop
-      with:
-        github-token: ${{ secrets.github_token }}
-        parallel-finished: true
+      - name: Coveralls Finished
+        uses: AndreMiras/coveralls-python-action@develop
+        with:
+          github-token: ${{ secrets.github_token }}
+          parallel-finished: true
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - name: flake8
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.9'
-    - name: install dependencies
-      run: python -m pip install --upgrade pip
-    - name: Check out repository code
-      uses: actions/checkout@v2
-    - name: lint
-      run: |
-        pip install -q flake8
-        flake8 qiita_* setup.py scripts/qiita* notebooks/*/*.py
+      - name: flake8
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+      - name: install dependencies
+        run: python -m pip install --upgrade pip
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: lint
+        run: |
+          pip install -q flake8
+          flake8 qiita_* setup.py scripts/qiita* notebooks/*/*.py

.gitignore

@@ -75,3 +75,6 @@ notebooks/*/*.tsv.gz
 
 # jupyter notebooks input data
 notebooks/resource-allocation/data
+
+# ignore plugin SQL injection file
+qiita_db/support_files/patches/test_db_sql/91.sql

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Qiita changelog
 
+Version 2025.07
+---------------
+
+Deployed on July 15th, 2025
+
+* During EBI-ENA submissions now we automatically renmae `country` to `geographic location (country and/or sea)` and `collection_date` to `collection date`; as the ENA requirements changed.
+* Added platform `DNBSEQ1` and instruments: `DNBSEQ-G400`, `DNBSEQ-T7`, `DNBSEQ-G800` for EBI-ENA submissions.
+* Other general fixes [#3474](https://github.com/qiita-spots/qiita/pull/3474), [#3475](https://github.com/qiita-spots/qiita/pull/3475).
+* SPP: merged and deprecated [mg-scripts](https://github.com/qiita-spots/mg-scripts) to [qp-knight-lab-processing](https://github.com/qiita-spots/qp-knight-lab-processing).
+* SPP: Added integration tests via a couple of PRs: [#129](https://github.com/qiita-spots/qp-knight-lab-processing/pull/129) & [#131](https://github.com/qiita-spots/qp-knight-lab-processing/pull/131).
+* SPP: Added new command `Human Filter & QC existing Prep` to facilitate human-filtering existing preparations.
+* SPP: Cleaned and centralized sequencers information to [kl-metapool](https://github.com/biocore/kl-metapool) & added `MiSeq i100`, thank you @AmandaBirmingham.
+* SPP: Added a new [subsample method](https://github.com/qiita-spots/qp-knight-lab-processing/pull/138) to randomly select sequences from paired files when they are larger than expected: 720,000,000.
+* SPP: Storing `TellReadJob/QC_Analysis_TellReadJob.html` in the results folder.
+
+
+
+Version 2025.04
+---------------
+
+Deployed on April 11th, 2025
+
+* General improvements for automatic environment generation by @sjanssen2 (thank you!): [#3462](https://github.com/qiita-spots/qiita/pull/3462), [#3463](https://github.com/qiita-spots/qiita/pull/3463), [3464](https://github.com/qiita-spots/qiita/pull/3464), [#3465](https://github.com/qiita-spots/qiita/pull/3465).
+* When ProcessingJob.resource_allocation_info fails, it will now be [set as the error for the job](https://github.com/qiita-spots/qiita/pull/3466).
+* SPP: General updates and clean up: [#169](https://github.com/biocore/mg-scripts/pull/169), [#101](https://github.com/qiita-spots/qp-knight-lab-processing/pull/101).
+* `Remove SynDNA plasmid, insert, & CP026085 reads` superseded `Remove SynDNA inserts & plasmid reads`; which now removes SynDNA plasmids, inserts, and CP026085 reads, in this order.
+
+
 Version 2025.02
 ---------------
 
@@ -8,7 +36,7 @@ Deployed on February 24th, 2025
 * Replaced os.rename for shutil.move in the code to fix [#3455](https://github.com/qiita-spots/qiita/issues/3455).
 * Via qp-spades, replaced the legacy `spades` command for `cloudSPAdes` for TellSeq.
 * `FASTA_preprocessed` within qtp-sequencing now allows for results to be named using their sample-name, extra from run-prefix.
-* `Remove SynDNA inserts & plasmid reads` superseded `Remove SynDNA reads`, which now removes SynDna inserts and plasmids.
+* `Remove SynDNA inserts & plasmid reads` superseded `Remove SynDNA reads`, which now removes SynDNA inserts and plasmids.
 * `update_resource_allocation_redis` now relies on using equations stored in the database vs. hardcoded; thank you @Gossty!
 * SPP: Updated prep-info file generation to identify and report filtered fastq files that could not be matched to a sample-id instead of silently ignoring them.
 * SPP: Removed legacy test code and example files for amplicon processing. Some other tests updated and repurposed.

qiita_core/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2025.02"
+__version__ = "2025.07"

qiita_core/support_files/config_test.cfg

@@ -25,9 +25,6 @@ REQUIRE_APPROVAL = True
 # Base URL: DO NOT ADD TRAILING SLASH
 BASE_URL = https://localhost:8383
 
-# Download path files
-UPLOAD_DATA_DIR = /home/runner/work/qiita/qiita/qiita_db/support_files/test_data/uploads/
-
 # Working directory path
 WORKING_DIR = /home/runner/work/qiita/qiita/qiita_db/support_files/test_data/working_dir/
 

qiita_core/tests/test_configuration_manager.py

@@ -306,9 +306,6 @@ def test_get_portal_latlong(self):
 # Base URL: DO NOT ADD TRAILING SLASH
 BASE_URL = https://localhost
 
-# Download path files
-UPLOAD_DATA_DIR = /tmp/
-
 # Working directory path
 WORKING_DIR = /tmp/
 

qiita_db/__init__.py

@@ -27,7 +27,7 @@
 from . import user
 from . import processing_job
 
-__version__ = "2025.02"
+__version__ = "2025.07"
 
 __all__ = ["analysis", "artifact",  "archive", "base", "commands",
            "environment_manager", "exceptions", "investigation", "logger",

qiita_db/environment_manager.py

@@ -7,6 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from os.path import abspath, dirname, join, exists, basename, splitext
+from shutil import copytree
 from functools import partial
 from os import mkdir
 import gzip
@@ -127,6 +128,36 @@ def _download_reference_files():
         _insert_processed_params(ref)
 
 
+def create_mountpoints():
+    r"""In a fresh qiita setup, sub-directories under
+        qiita_config.base_data_dir might not yet exist. To avoid failing in
+        later steps, they are created here.
+    """
+    with qdb.sql_connection.TRN:
+        sql = """SELECT DISTINCT mountpoint FROM qiita.data_directory
+                 WHERE active = TRUE"""
+        qdb.sql_connection.TRN.add(sql)
+        created_subdirs = []
+        for mountpoint in qdb.sql_connection.TRN.execute_fetchflatten():
+            for (ddid, subdir) in qdb.util.get_mountpoint(mountpoint,
+                                                          retrieve_all=True):
+                if not exists(join(qiita_config.base_data_dir, subdir)):
+                    if qiita_config.test_environment:
+                        # if in test mode, we want to potentially fill the
+                        # new directory with according test data
+                        copytree(get_support_file('test_data', mountpoint),
+                                 join(qiita_config.base_data_dir, subdir))
+                    else:
+                        # in production mode, an empty directory is created
+                        mkdir(join(qiita_config.base_data_dir, subdir))
+                    created_subdirs.append(subdir)
+
+        if len(created_subdirs) > 0:
+            print("Created %i sub-directories as 'mount points':\n%s"
+                  % (len(created_subdirs),
+                     ''.join(map(lambda x: ' - %s\n' % x, created_subdirs))))
+
+
 def make_environment(load_ontologies, download_reference, add_demo_user):
     r"""Creates the new environment specified in the configuration
 
@@ -397,6 +428,9 @@ def patch(patches_dir=PATCHES_DIR, verbose=False, test=False):
         with qdb.sql_connection.TRN:
             _populate_test_db()
 
+    # create mountpoints as subdirectories in BASE_DATA_DIR
+    create_mountpoints()
+
     patch_update_sql = "UPDATE settings SET current_patch = %s"
     for sql_patch_fp in sql_patch_files[next_patch_index:]:
         sql_patch_filename = basename(sql_patch_fp)

qiita_db/handlers/tests/oauthbase.py

@@ -6,7 +6,10 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-from qiita_core.qiita_settings import r_client
+import requests
+import os
+import sys
+from qiita_core.qiita_settings import r_client, qiita_config
 
 from qiita_pet.test.tornado_test_base import TestHandlerBase
 
@@ -19,3 +22,49 @@ def setUp(self):
         r_client.hset(self.token, 'grant_type', 'client')
         r_client.expire(self.token, 20)
         super(OauthTestingBase, self).setUp()
+        self._session = requests.Session()
+        # should point to client certificat file:
+        # /qiita/qiita_core/support_files/ci_rootca.crt
+        self._verify = os.environ['QIITA_ROOTCA_CERT']
+        self._fetch_token()
+
+        self._files_to_remove = []
+
+    def tearDown(self):
+        for fp in self._files_to_remove:
+            if os.path.exists(fp):
+                os.remove(fp)
+
+    def _fetch_token(self):
+        data = {
+            'client_id': '4MOBzUBHBtUmwhaC258H7PS0rBBLyGQrVxGPgc9g305bvVhf6h',
+            'client_secret':
+            ('rFb7jwAb3UmSUN57Bjlsi4DTl2owLwRpwCc0SggRN'
+             'EVb2Ebae2p5Umnq20rNMhmqN'),
+            'grant_type': 'client'}
+        resp = self._session.post(
+            "%s/qiita_db/authenticate/" % qiita_config.base_url,
+            verify=self._verify, data=data, timeout=80)
+        if resp.status_code != 200:
+            raise ValueError("_fetchToken() POST request failed")
+        self._token = resp.json()['access_token']
+        print('obtained access_token = %s' % self._token, file=sys.stderr)
+
+    def post_authed(self, url, **kwargs):
+        if 'headers' not in kwargs:
+            kwargs['headers'] = {}
+        if 'Authorization' not in kwargs['headers']:
+            kwargs['headers']['Authorization'] = 'Bearer %s' % self._token
+
+        r = self._session.post(
+            qiita_config.base_url + url, verify=self._verify, **kwargs)
+        r.close()
+
+        return r
+
+    def get_authed(self, url):
+        r = self._session.get(qiita_config.base_url + url, verify=self._verify,
+                              headers={'Authorization': 'Bearer %s' %
+                                       self._token})
+        r.close()
+        return r

qiita_db/metadata_template/test/test_base_metadata_template.py

@@ -125,12 +125,13 @@ def test_restrictions(self):
         obs = MT.prep_template.PrepTemplate(1).restrictions
         exp = {
             'target_gene': ['16S rRNA', '18S rRNA', 'ITS1/2', 'LSU'],
-            'platform': ['FASTA', 'Illumina', 'Ion_Torrent', 'LS454',
+            'platform': ['DNBSEQ', 'FASTA', 'Illumina', 'Ion_Torrent', 'LS454',
                          'Oxford Nanopore'],
             'target_subfragment': ['V3', 'V4', 'V6', 'V9', 'ITS1/2'],
             'instrument_model': [
                 '454 GS', '454 GS 20', '454 GS FLX', '454 GS FLX+',
-                '454 GS FLX Titanium', '454 GS Junior',
+                '454 GS FLX Titanium', '454 GS Junior', 'DNBSEQ-G400',
+                'DNBSEQ-T7', 'DNBSEQ-G800',
                 'Illumina Genome Analyzer', 'Illumina Genome Analyzer II',
                 'Illumina Genome Analyzer IIx', 'Illumina HiScanSQ',
                 'Illumina HiSeq 1000', 'Illumina HiSeq 1500',