Merge branch 'dev' of https://github.com/qiita-spots/qiita into dev

Author: Gossty

.github/workflows/qiita-ci.yml

@@ -104,9 +104,7 @@ jobs:
       - name: Install plugins
         shell: bash -l {0}
         run: |
-          wget https://data.qiime2.org/distro/core/qiime2-2022.11-py38-linux-conda.yml
-          conda env create --quiet -n qtp-biom --file qiime2-2022.11-py38-linux-conda.yml
-          rm qiime2-2022.11-py38-linux-conda.yml
+          conda env create -n qtp-biom --file https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml
           export QIITA_ROOTCA_CERT=`pwd`/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita_core/support_files/config_test.cfg
           export REDBIOM_HOST="http://localhost:7379"
@@ -181,7 +179,7 @@ jobs:
           echo "Connecting as $USER@localhost"
           # this line (and the -o StrictHostKeyChecking=no) is so the server
           # is added to the list of known servers
-          scp -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
+          scp -O -o StrictHostKeyChecking=no -i $PWD/qiita_ware/test/test_data/test_key $USER@localhost:/home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key /home/runner/work/qiita/qiita/qiita_ware/test/test_data/random_key_copy_1
 
       - name: Main tests
         shell: bash -l {0}

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Qiita changelog
 
+Version 2024.10
+---------------
+
+Deployed on October 14th, 2024
+
+* Added update_resource_allocation_redis and companion code, so resource allocations summaries are available for review. Thank you @Gossty!
+* Now is possible to have default workflows with only one step.
+* `qiita_client.update_job_step` now accepts an ignore_error optional parameter. Thank you @charles-cowart!
+* Initial changes in `qiita_client` to have more accurate variable names: `QIITA_SERVER_CERT` -> `QIITA_ROOTCA_CERT`. Thank you @charles-cowart!
+* Added `get_artifact_html_summary` to `qiita_client` to retrieve the summary file of an artifact.
+* Re-added github actions to `https://github.com/qiita-spots/qiita_client`.
+* `SortMeRNA v4.3.7` superseded `Sortmerna v2.1b`, which relies on Silva 138 and now produced even mates. Thank you @ekopylova and @biocodz for the support.
+* `Remove SynDNA reads` superseded `SynDNA Woltka`, which now generates even mates.
+* `Woltka v0.1.7, paired-end` superseded `Woltka v0.1.6` in `qp-woltka`; [more information](https://qiita.ucsd.edu/static/doc/html/processingdata/woltka_pairedend.html). Thank you to @qiyunzhu for the benchmarks!
+* Other general fixes, like [#3424](https://github.com/qiita-spots/qiita/pull/3424), [#3425](https://github.com/qiita-spots/qiita/pull/3425), [#3439](https://github.com/qiita-spots/qiita/pull/3439), [#3440](https://github.com/qiita-spots/qiita/pull/3440).
+* General SPP improvements, like: [NuQC modified to preserve metadata in fastq files](https://github.com/biocore/mg-scripts/pull/155), [use squeue instead of sacct](https://github.com/biocore/mg-scripts/pull/152), , [job aborts if Qiita study contains sample metadata columns reserved for prep-infos](https://github.com/biocore/mg-scripts/pull/151), [metapool generates OverrideCycles value](https://github.com/biocore/metagenomics_pooling_notebook/pull/225).
+* We updated the available parameters for `Filter features against reference [filter_features]`, `Non V4 16S sequence assessment [non_v4_16s]` and all the phylogenetic analytical commands so they can use `Greengenes2 2024.09`.
+
+
+
 Version 2024.07
 ---------------
 

qiita_core/__init__.py

@@ -6,4 +6,4 @@
 # The full license is in the file LICENSE, distributed with this software.
 # -----------------------------------------------------------------------------
 
-__version__ = "2024.02"
+__version__ = "2024.10"

qiita_db/__init__.py

@@ -27,7 +27,7 @@
 from . import user
 from . import processing_job
 
-__version__ = "2024.02"
+__version__ = "2024.10"
 
 __all__ = ["analysis", "artifact",  "archive", "base", "commands",
            "environment_manager", "exceptions", "investigation", "logger",

qiita_db/analysis.py

@@ -215,6 +215,22 @@ def create(cls, owner, name, description, from_default=False,
         job.submit()
         return instance
 
+    @classmethod
+    def delete_analysis_artifacts(cls, _id):
+        """Deletes the artifacts linked to an artifact and then the analysis
+
+        Parameters
+        ----------
+        _id : int
+            The analysis id
+        """
+        analysis = cls(_id)
+        aids = [a.id for a in analysis.artifacts if not a.parents]
+        aids.sort(reverse=True)
+        for aid in aids:
+            qdb.artifact.Artifact.delete(aid)
+        cls.delete(analysis.id)
+
     @classmethod
     def delete(cls, _id):
         """Deletes an analysis

qiita_db/artifact.py

@@ -929,7 +929,8 @@ def can_be_submitted_to_ebi(self):
             # words has more that one processing step behind it
             fine_to_send = []
             fine_to_send.extend([pt.artifact for pt in self.prep_templates])
-            fine_to_send.extend([c for a in fine_to_send for c in a.children])
+            fine_to_send.extend([c for a in fine_to_send if a is not None
+                                 for c in a.children])
             if self not in fine_to_send:
                 return False
 
@@ -1342,23 +1343,6 @@ def _helper(sql_edges, edges, nodes):
                         # If the job is in success we don't need to do anything
                         # else since it would've been added by the code above
                         if jstatus != 'success':
-                            # Connect the job with his input artifacts, the
-                            # input artifacts may or may not exist yet, so we
-                            # need to check both the input_artifacts and the
-                            # pending properties
-                            for in_art in n_obj.input_artifacts:
-                                iid = in_art.id
-                                if iid not in nodes and iid in extra_nodes:
-                                    nodes[iid] = extra_nodes[iid]
-                                _add_edge(edges, nodes[iid], nodes[n_obj.id])
-
-                            pending = n_obj.pending
-                            for pred_id in pending:
-                                for pname in pending[pred_id]:
-                                    in_node_id = '%s:%s' % (
-                                        pred_id, pending[pred_id][pname])
-                                    _add_edge(edges, nodes[in_node_id],
-                                              nodes[n_obj.id])
 
                             if jstatus != 'error':
                                 # If the job is not errored, we can add the
@@ -1380,6 +1364,34 @@ def _helper(sql_edges, edges, nodes):
                                     queue.append(cjob.id)
                                     if cjob.id not in nodes:
                                         nodes[cjob.id] = ('job', cjob)
+
+                                    # including the outputs
+                                    for o_name, o_type in cjob.command.outputs:
+                                        node_id = '%s:%s' % (cjob.id, o_name)
+                                        node = TypeNode(
+                                            id=node_id, job_id=cjob.id,
+                                            name=o_name, type=o_type)
+                                        if node_id not in nodes:
+                                            nodes[node_id] = ('type', node)
+
+                            # Connect the job with his input artifacts, the
+                            # input artifacts may or may not exist yet, so we
+                            # need to check both the input_artifacts and the
+                            # pending properties
+                            for in_art in n_obj.input_artifacts:
+                                iid = in_art.id
+                                if iid not in nodes and iid in extra_nodes:
+                                    nodes[iid] = extra_nodes[iid]
+                                _add_edge(edges, nodes[iid], nodes[n_obj.id])
+
+                            pending = n_obj.pending
+                            for pred_id in pending:
+                                for pname in pending[pred_id]:
+                                    in_node_id = '%s:%s' % (
+                                        pred_id, pending[pred_id][pname])
+                                    _add_edge(edges, nodes[in_node_id],
+                                              nodes[n_obj.id])
+
                     elif n_type == 'type':
                         # Connect this 'future artifact' with the job that will
                         # generate it

qiita_db/handlers/processing_job.py

@@ -146,7 +146,9 @@ def post(self, job_id):
                 cmd, values_dict={'job_id': job_id,
                                   'payload': self.request.body.decode(
                                       'ascii')})
-            job = qdb.processing_job.ProcessingJob.create(job.user, params)
+            # complete_job are unique so it is fine to force them to be created
+            job = qdb.processing_job.ProcessingJob.create(
+                job.user, params, force=True)
             job.submit()
 
         self.finish()

qiita_db/handlers/tests/test_processing_job.py

@@ -233,9 +233,9 @@ def test_post_job_success(self):
         self.assertIsNotNone(cj)
         # additionally we can test that job.print_trace is correct
         self.assertEqual(job.trace, [
-            f'{job.id} [Not Available]: Validate | '
+            f'{job.id} [Not Available] (success): Validate | '
             '-p qiita -N 1 -n 1 --mem 90gb --time 150:00:00 --nice=10000',
-            f'  {cj.id} [{cj.external_id}] | '
+            f'  {cj.id} [{cj.external_id}] (success)| '
             '-p qiita -N 1 -n 1 --mem 16gb --time 10:00:00 --nice=10000'])
 
     def test_post_job_success_with_archive(self):

qiita_db/meta_util.py

@@ -37,11 +37,18 @@
 from re import sub
 from json import loads, dump, dumps
 
-from qiita_db.util import create_nested_path
+from qiita_db.util import create_nested_path, retrieve_resource_data
+from qiita_db.util import resource_allocation_plot
 from qiita_core.qiita_settings import qiita_config, r_client
 from qiita_core.configuration_manager import ConfigurationManager
 import qiita_db as qdb
 
+# global constant list used in resource_allocation_page
+COLUMNS = [
+    "sName", "sVersion", "cID", "cName", "processing_job_id",
+    "parameters", "samples", "columns", "input_size", "extra_info",
+    "MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"]
+
 
 def _get_data_fpids(constructor, object_id):
     """Small function for getting filepath IDS associated with data object
@@ -546,3 +553,101 @@ def generate_plugin_releases():
         # important to "flush" variables to avoid errors
         r_client.delete(redis_key)
         f(redis_key, v)
+
+
+def get_software_commands(active):
+    software_list = [s for s in qdb.software.Software.iter(active=active)]
+    software_commands = defaultdict(lambda: defaultdict(list))
+
+    for software in software_list:
+        sname = software.name
+        sversion = software.version
+        commands = software.commands
+
+        for command in commands:
+            software_commands[sname][sversion].append(command.name)
+        software_commands[sname] = dict(software_commands[sname])
+
+    return dict(software_commands)
+
+
+def update_resource_allocation_redis(active=True):
+    """Updates redis with plots and information about current software.
+
+    Parameters
+    ----------
+    active: boolean, optional
+        Defaults to True. Should only be False when testing.
+
+    """
+    time = datetime.now().strftime('%m-%d-%y')
+    scommands = get_software_commands(active)
+    redis_key = 'resources:commands'
+    r_client.set(redis_key, str(scommands))
+
+    for sname, versions in scommands.items():
+        for version, commands in versions.items():
+            for cname in commands:
+                col_name = "samples * columns"
+                df = retrieve_resource_data(cname, sname, version, COLUMNS)
+                if len(df) == 0:
+                    continue
+
+                fig, axs = resource_allocation_plot(df, cname, sname, col_name)
+                titles = [0, 0]
+                images = [0, 0]
+
+                # Splitting 1 image plot into 2 separate for better layout.
+                for i, ax in enumerate(axs):
+                    titles[i] = ax.get_title()
+                    ax.set_title("")
+                    # new_fig, new_ax – copy with either only memory plot or
+                    # only time
+                    new_fig = plt.figure()
+                    new_ax = new_fig.add_subplot(111)
+
+                    scatter_data = ax.collections[0]
+                    new_ax.scatter(scatter_data.get_offsets()[:, 0],
+                                   scatter_data.get_offsets()[:, 1],
+                                   s=scatter_data.get_sizes(), label="data")
+
+                    line = ax.lines[0]
+                    new_ax.plot(line.get_xdata(), line.get_ydata(),
+                                linewidth=1, color='orange')
+
+                    if len(ax.collections) > 1:
+                        failure_data = ax.collections[1]
+                        new_ax.scatter(failure_data.get_offsets()[:, 0],
+                                       failure_data.get_offsets()[:, 1],
+                                       color='red', s=3, label="failures")
+
+                    new_ax.set_xscale('log')
+                    new_ax.set_yscale('log')
+                    new_ax.set_xlabel(ax.get_xlabel())
+                    new_ax.set_ylabel(ax.get_ylabel())
+                    new_ax.legend(loc='upper left')
+
+                    new_fig.tight_layout()
+                    plot = BytesIO()
+                    new_fig.savefig(plot, format='png')
+                    plot.seek(0)
+                    img = 'data:image/png;base64,' + quote(
+                          b64encode(plot.getvalue()).decode('ascii'))
+                    images[i] = img
+                    plt.close(new_fig)
+                plt.close(fig)
+
+                # SID, CID, col_name
+                values = [
+                    ("img_mem", images[0], r_client.set),
+                    ("img_time", images[1], r_client.set),
+                    ('time', time, r_client.set),
+                    ("title_mem", titles[0], r_client.set),
+                    ("title_time", titles[1], r_client.set)
+                ]
+
+                for k, v, f in values:
+                    redis_key = 'resources$#%s$#%s$#%s$#%s:%s' % (
+                                cname, sname, version, col_name, k)
+                    r_client.delete(redis_key)
+                    f(redis_key, v)

qiita_db/metadata_template/prep_template.py

@@ -135,7 +135,7 @@ def create(cls, md_template, study, data_type, investigation_type=None,
             # data_type being created - if possible
             if investigation_type is None:
                 if data_type_str in TARGET_GENE_DATA_TYPES:
-                    investigation_type = 'Amplicon'
+                    investigation_type = 'AMPLICON'
                 elif data_type_str == 'Metagenomic':
                     investigation_type = 'WGS'
                 elif data_type_str == 'Metatranscriptomic':
@@ -272,6 +272,32 @@ def delete(cls, id_):
                     "Cannot remove prep template %d because it has an artifact"
                     " associated with it" % id_)
 
+            # artifacts that are archived are not returned as part of the code
+            # above and we need to clean them before moving forward
+            sql = """SELECT artifact_id
+                     FROM qiita.preparation_artifact
+                     WHERE prep_template_id = %s"""
+            qdb.sql_connection.TRN.add(sql, args)
+            archived_artifacts = set(
+                qdb.sql_connection.TRN.execute_fetchflatten())
+            ANALYSIS = qdb.analysis.Analysis
+            if archived_artifacts:
+                for aid in archived_artifacts:
+                    # before we can delete the archived artifact, we need
+                    # to delete the analyses where they were used.
+                    sql = """SELECT analysis_id
+                             FROM qiita.analysis
+                             WHERE analysis_id IN (
+                                SELECT DISTINCT analysis_id
+                                FROM qiita.analysis_sample
+                                WHERE artifact_id IN %s)"""
+                    qdb.sql_connection.TRN.add(sql, [tuple([aid])])
+                    analyses = set(
+                        qdb.sql_connection.TRN.execute_fetchflatten())
+                    for _id in analyses:
+                        ANALYSIS.delete_analysis_artifacts(_id)
+                    qdb.artifact.Artifact.delete(aid)
+
             # Delete the prep template filepaths
             sql = """DELETE FROM qiita.prep_template_filepath
                      WHERE prep_template_id = %s"""
@@ -815,6 +841,9 @@ def _get_predecessors(workflow, node):
                 pred.append(data)
                 return pred
 
+            # this is only helpful for when there are no _get_predecessors
+            return pred
+
         # Note: we are going to use the final BIOMs to figure out which
         #       processing is missing from the back/end to the front, as this
         #       will prevent generating unnecessary steps (AKA already provided
@@ -937,6 +966,8 @@ def _get_predecessors(workflow, node):
                     if set(merging_schemes[info]) >= set(cxns):
                         init_artifacts = merging_schemes[info]
                         break
+            if not predecessors:
+                pnode = node
             if init_artifacts is None:
                 pdp = pnode.default_parameter
                 pdp_cmd = pdp.command

qiita_db/metadata_template/test/test_prep_template.py

@@ -911,7 +911,7 @@ def _common_creation_checks(self, pt, fp_count, name):
         self.assertEqual(pt.data_type(), self.data_type)
         self.assertEqual(pt.data_type(ret_id=True), self.data_type_id)
         self.assertEqual(pt.artifact, None)
-        self.assertEqual(pt.investigation_type, 'Amplicon')
+        self.assertEqual(pt.investigation_type, 'AMPLICON')
         self.assertEqual(pt.study_id, self.test_study.id)
         self.assertEqual(pt.status, "sandbox")
         exp_sample_ids = {'%s.SKB8.640193' % self.test_study.id,
@@ -1076,7 +1076,7 @@ def test_create_warning(self):
         self.assertEqual(pt.data_type(), self.data_type)
         self.assertEqual(pt.data_type(ret_id=True), self.data_type_id)
         self.assertEqual(pt.artifact, None)
-        self.assertEqual(pt.investigation_type, 'Amplicon')
+        self.assertEqual(pt.investigation_type, 'AMPLICON')
         self.assertEqual(pt.study_id, self.test_study.id)
         self.assertEqual(pt.status, 'sandbox')
         exp_sample_ids = {'%s.SKB8.640193' % self.test_study.id,
@@ -1247,7 +1247,7 @@ def test_investigation_type_setter(self):
         """Able to update the investigation type"""
         pt = qdb.metadata_template.prep_template.PrepTemplate.create(
             self.metadata, self.test_study, self.data_type_id)
-        self.assertEqual(pt.investigation_type, 'Amplicon')
+        self.assertEqual(pt.investigation_type, 'AMPLICON')
         pt.investigation_type = "Other"
         self.assertEqual(pt.investigation_type, 'Other')
         with self.assertRaises(qdb.exceptions.QiitaDBColumnError):