Merge branch 'dev' of github.com:biocore/qiita into dev

Author: antgonza

INSTALL.md

@@ -78,7 +78,7 @@ You can reboot the system with `sudo reboot` in case any packages were updated.
 Next, we need to add the Postgres repository to our system:
 ```bash
 sudo apt update
-sudo apt install curl gpg gnupg2 software-properties-common apt-transport-https lsb-release ca-certificates
+sudo apt install curl gpg gnupg2 software-properties-common apt-transport-https lsb-release ca-certificates git
 curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc|sudo gpg --dearmor -o /etc/apt/trusted.gpg.d/postgresql.gpg
 echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" |sudo tee  /etc/apt/sources.list.d/pgdg.list
 ```
@@ -238,6 +238,9 @@ If you are using [NGINX](https://www.nginx.com/) via conda, you are going to nee
 mkdir -p ${CONDA_PREFIX}/var/run/nginx/
 ```
 
+Note that the shipped nginx version from conda, does **not** contain the mod_zip module: https://github.com/evanmiller/mod_zip
+This leads to unexpected behaviour when generating a download link for anonymous artefact sharing, i.e. Qiita returns a flat file listing artifact filepaths instead of generating a ZIP archive that contains those files. You need to compile nginx with the additional mod_zip module yourself. (I've invested multiple hours to realize that the configure routine does not properly link shared libraries to the nginx binary. Try adding `--with-ld-opt=" -Wl,-rpath,/home/foo/lib "` to the `./auto/configure` call.)
+
 ## Start Qiita
 
 Start postgres (instructions vary depending on operating system and install method).

qiita_core/configuration_manager.py

@@ -389,4 +389,4 @@ def _get_portal(self, config):
                 raise ValueError(msg % (name, val, 'larger than 180°'))
 
     def _iframe(self, config):
-        self.iframe_qiimp = config.get('iframe', 'QIIMP')
+        self.iframe_qiimp = config.get('iframe', 'QIIMP', fallback=None)

qiita_core/support_files/config_test.cfg

@@ -193,6 +193,6 @@ STATS_MAP_CENTER_LONGITUDE =
 
 # ----------------------------- iframes settings ---------------------------
 [iframe]
-# The real world QIIMP will always need to be accessed with https because Qiita
-# runs on https too
-QIIMP = https://localhost:8898/
+# On May 2024, we removed QIIMP from the code base but we will leave this
+# section in case we need to add access to another iframe in the future; note
+# that the qiita-terms are also accessed via iframe but this is internal

qiita_core/tests/test_configuration_manager.py

@@ -109,7 +109,7 @@ def test_init(self):
         self.assertEqual(obs.portal_dir, "/portal")
 
         # iframe section
-        self.assertEqual(obs.iframe_qiimp, "https://localhost:8898/")
+        self.assertIsNone(obs.iframe_qiimp)
 
     def test_init_error(self):
         with open(self.conf_fp, 'w') as f:
@@ -471,7 +471,6 @@ def test_get_portal_latlong(self):
 
 # ----------------------------- iframes settings ---------------------------
 [iframe]
-QIIMP = https://localhost:8898/
 """
 
 if __name__ == '__main__':

qiita_db/support_files/patches/test_db_sql/92.sql

@@ -286,7 +286,7 @@ def _get_daily_stats():
             ('num_studies_ebi', b'1', r_client.get),
             ('num_samples_ebi', b'27', r_client.get),
             ('number_samples_ebi_prep', b'54', r_client.get),
-            ('num_processing_jobs', b'14', r_client.get)
+            ('num_processing_jobs', b'474', r_client.get)
             # not testing img/time for simplicity
             # ('img', r_client.get),
             # ('time', r_client.get)

qiita_db/test/test_meta_util.py

@@ -494,13 +494,20 @@ def test_processing_jobs(self):
                     '6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f',
                     '063e553b-327c-4818-ab4a-adfe58e49860',
                     'ac653cb5-76a6-4a45-929e-eb9b2dee6b63']
-        exp = [qdb.processing_job.ProcessingJob(j) for j in exp_jids]
-        self.assertCountEqual(qdb.software.Command(1).processing_jobs, exp)
+
+        jobs = qdb.software.Command(1).processing_jobs
+        set_jobs = set(jobs)
+
+        # comparing the length of jobs and set_jobs, since there could've been
+        # duplicates in the tests
+        self.assertEqual(len(jobs), len(set_jobs))
+
+        exp = set([qdb.processing_job.ProcessingJob(j) for j in exp_jids])
+        self.assertEqual(len(set_jobs & exp), len(exp_jids))
 
         exp_jids = ['bcc7ebcd-39c1-43e4-af2d-822e3589f14d']
         exp = [qdb.processing_job.ProcessingJob(j) for j in exp_jids]
         self.assertCountEqual(qdb.software.Command(2).processing_jobs, exp)
-
         self.assertCountEqual(qdb.software.Command(4).processing_jobs, [])
 
 

qiita_db/test/test_software.py

@@ -491,17 +491,10 @@ def test_jobs(self):
             limit=1, ignore_status=ignore_status), [
                 PJ('b72369f9-a886-4193-8d3d-f7b504168e75')])
 
-        # no jobs
-        self.assertEqual(qdb.user.User('[email protected]').jobs(
-            ignore_status=ignore_status), [])
-
         # generates expected jobs
         jobs = qdb.user.User('[email protected]').jobs()
         self.assertEqual(jobs, [])
 
-        # no jobs
-        self.assertEqual(qdb.user.User('[email protected]').jobs(), [])
-
     def test_update_email(self):
         user = qdb.user.User('[email protected]')
         with self.assertRaisesRegex(IncorrectEmailError, 'Bad email given:'):

qiita_db/test/test_user.py

@@ -1309,19 +1309,23 @@ def test_quick_mounts_purge(self):
 
 class ResourceAllocationPlotTests(TestCase):
     def setUp(self):
-
-        self.PATH_TO_DATA = ('./qiita_db/test/test_data/'
-                             'jobs_2024-02-21.tsv.gz')
-        self.CNAME = "Validate"
-        self.SNAME = "Diversity types - alpha_vector"
+        self.CNAME = "Split libraries FASTQ"
+        self.SNAME = "QIIMEq2"
         self.col_name = 'samples * columns'
-        self.df = pd.read_csv(self.PATH_TO_DATA, sep='\t',
-                              dtype={'extra_info': str})
+        self.columns = [
+                "sName", "sVersion", "cID", "cName", "processing_job_id",
+                "parameters", "samples", "columns", "input_size", "extra_info",
+                "MaxRSSRaw", "ElapsedRaw"]
+
+        # df is a dataframe that represents a table with columns specified in
+        # self.columns
+        self.df = qdb.util._retrieve_resource_data(
+                self.CNAME, self.SNAME, self.columns)
 
     def test_plot_return(self):
         # check the plot returns correct objects
         fig1, axs1 = qdb.util.resource_allocation_plot(
-            self.PATH_TO_DATA, self.CNAME, self.SNAME, self.col_name)
+            self.df, self.CNAME, self.SNAME, self.col_name)
         self.assertIsInstance(
             fig1, Figure,
             "Returned object fig1 is not a Matplotlib Figure")
@@ -1346,7 +1350,7 @@ def test_minimize_const(self):
         failures_df = qdb.util._resource_allocation_failures(
             self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')
         failures = failures_df.shape[0]
-        self.assertEqual(bm, qdb.util.mem_model4, msg="""Best memory model
+        self.assertEqual(bm, qdb.util.mem_model3, msg="""Best memory model
                                                 doesn't match""")
         self.assertEqual(failures, 0, "Number of failures must be 0")
 

qiita_db/test/test_util.py

@@ -73,10 +73,10 @@
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
-import pandas as pd
 from datetime import timedelta
 import matplotlib.pyplot as plt
 import numpy as np
+import pandas as pd
 from scipy.optimize import minimize
 
 # memory constant functions defined for @resource_allocation_plot
@@ -2341,7 +2341,7 @@ def send_email(to, subject, body):
         smtp.close()
 
 
-def resource_allocation_plot(file, cname, sname, col_name):
+def resource_allocation_plot(df, cname, sname, col_name):
     """Builds resource allocation plot for given filename and jobs
 
     Parameters
@@ -2361,9 +2361,6 @@ def resource_allocation_plot(file, cname, sname, col_name):
         Returns a matplotlib object with a plot
     """
 
-    df = pd.read_csv(file, sep='\t', dtype={'extra_info': str})
-    df['ElapsedRawTime'] = pd.to_timedelta(df.ElapsedRawTime)
-    df = df[(df.cName == cname) & (df.sName == sname)]
     df.dropna(subset=['samples', 'columns'], inplace=True)
     df[col_name] = df.samples * df['columns']
     df[col_name] = df[col_name].astype(int)
@@ -2383,6 +2380,41 @@ def resource_allocation_plot(file, cname, sname, col_name):
     return fig, axs
 
 
+def _retrieve_resource_data(cname, sname, columns):
+    with qdb.sql_connection.TRN:
+        sql = """
+            SELECT
+                s.name AS sName,
+                s.version AS sVersion,
+                sc.command_id AS cID,
+                sc.name AS cName,
+                pr.processing_job_id AS processing_job_id,
+                pr.command_parameters AS parameters,
+                sra.samples AS samples,
+                sra.columns AS columns,
+                sra.input_size AS input_size,
+                sra.extra_info AS extra_info,
+                sra.memory_used AS memory_used,
+                sra.walltime_used AS walltime_used
+            FROM
+                qiita.processing_job pr
+            JOIN
+                qiita.software_command sc ON pr.command_id = sc.command_id
+            JOIN
+                qiita.software s ON sc.software_id = s.software_id
+            JOIN
+                qiita.slurm_resource_allocations sra
+                ON pr.processing_job_id = sra.processing_job_id
+            WHERE
+                sc.name = %s
+                AND s.name = %s;
+            """
+        qdb.sql_connection.TRN.add(sql, sql_args=[cname, sname])
+        res = qdb.sql_connection.TRN.execute_fetchindex()
+        df = pd.DataFrame(res, columns=columns)
+        return df
+
+
 def _resource_allocation_plot_helper(
         df, ax, cname, sname, curr, models, col_name):
     """Helper function for resource allocation plot. Builds plot for MaxRSSRaw

qiita_db/util.py

@@ -27,7 +27,6 @@ Looking for information about getting started? Please see the document here:
 
    gettingstartedguide/index.rst
    qiita-philosophy/index.rst
-   qiimp.rst
 
 Looking for information about how to process your data? Please see the document here:
 

qiita_pet/support_files/doc/source/index.rst

@@ -3,8 +3,6 @@
 
 {% if iframe == 'qiita-terms' %}
   <iframe style="margin: 0; padding: 0; width: 100%; height: 800px;" src="{% raw qiita_config.portal_dir %}/static/qiita_data_terms_of_use.html"></iframe>
-{% elif iframe == 'qiimp' %}
-  <iframe style="margin: 0; padding: 0; width: 100%; height: 800px;" src="{{qiita_config.iframe_qiimp}}"></iframe>
 {% else %}
   <b>No content</b>
 {% end %}

qiita_pet/support_files/doc/source/qiimp.rst

@@ -464,11 +464,6 @@
                 <a href="/redbiom/">redbiom</a>
               </li>
             </ul>
-            <ul class="nav navbar-nav">
-              <li>
-                <a href="{% raw qiita_config.portal_dir %}/iframe/?iframe=qiimp">Qiimp</a>
-              </li>
-            </ul>
             <ul class="nav navbar-nav">
               <!-- downloads -->
               <li class="dropdown">