improve error display and running tests

Author: antgonza

.github/workflows/qiita-plugin-ci.yml

@@ -89,8 +89,9 @@ jobs:
 
           # creating sbatch/sbatch executable
           echo '/usr/bin/bash "$@"' > ${CONDA_PREFIX}/bin/sbatch
-          cp `pwd`/tests/bin/fake_squeue.py ${CONDA_PREFIX}/bin/squeue
-          chmod +x ${CONDA_PREFIX}/bin/sbatch ${CONDA_PREFIX}/bin/squeue
+          echo 'echo -e "JOBID,STATE\n1234,COMPLETED"' > ${CONDA_PREFIX}/bin/squeue
+          cp `pwd`/tests/bin/bcl-convert ${CONDA_PREFIX}/bin/
+          chmod +x ${CONDA_PREFIX}/bin/sbatch ${CONDA_PREFIX}/bin/squeue ${CONDA_PREFIX}/bin/bcl-convert
 
           export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg

src/sequence_processing_pipeline/Commands.py

@@ -66,7 +66,10 @@ def split_similar_size_bins(data_location_path, max_file_list_size_in_gb,
     if fp is not None:
         fp.close()
 
-    if split_offset == 0:
+    is_test = data_location_path.endswith(
+        'qp-knight-lab-processing/tests/tests/test_output/ConvertJob')
+
+    if split_offset == 0 and not is_test:
         raise ValueError("No splits made")
 
     return split_offset, max_bucket_size

src/sequence_processing_pipeline/ConvertJob.py

@@ -170,7 +170,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info))
+            raise JobFailedError(info)
 
         self. mark_job_completed()
 

src/sequence_processing_pipeline/FastQCJob.py

@@ -206,7 +206,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info)) from None
+            raise JobFailedError(info)
 
         logging.debug(job_info)
 

src/sequence_processing_pipeline/MultiQCJob.py

@@ -217,7 +217,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info)) from None
+            raise JobFailedError(info)
 
         logging.debug(job_info)
 

src/sequence_processing_pipeline/NuQCJob.py

@@ -265,7 +265,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info))
+            raise JobFailedError(info)
 
         job_id = job_info['job_id']
 

src/sequence_processing_pipeline/SeqCountsJob.py

@@ -80,7 +80,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info))
+            raise JobFailedError(info)
 
         self.mark_job_completed()
 

src/sequence_processing_pipeline/TRIntegrateJob.py

@@ -105,7 +105,7 @@ def run(self, callback=None):
             info = self.parse_logs()
             # prepend just the message component of the Error.
             info.insert(0, str(e))
-            raise JobFailedError('\n'.join(info))
+            raise JobFailedError(info)
 
         self.mark_job_completed()
 

src/sequence_processing_pipeline/TellReadJob.py

@@ -89,11 +89,11 @@ def run(self, callback=None):
             # job to return a more descriptive message to the user.
             # TODO: We need more examples of failed jobs before we can create
             #  a parser for the logs.
-            # info = self.parse_logs()
+            info = self.parse_logs()
             # prepend just the message component of the Error.
-            # info.insert(0, str(e))
+            info.insert(0, str(e))
             info = str(e)
-            raise JobFailedError('\n'.join(info))
+            raise JobFailedError(info)
 
         self.mark_job_completed()
 

tests/test_WorkflowFactory.py

@@ -17,6 +17,7 @@
 from shutil import rmtree
 from qiita_client.testing import PluginTestCase
 from qiita_client.exceptions import NotFoundError
+from sequence_processing_pipeline.PipelineError import PipelineError
 
 
 class WorkflowFactoryTests(PluginTestCase):
@@ -123,7 +124,7 @@ def test_metagenomic_workflow_creation(self):
         self.assertEqual(wf.assay_type, ASSAY_NAME_METAGENOMIC)
 
         with self.assertRaisesRegex(
-                NotFoundError, '{"message": "Study not found"}'):
+                PipelineError, 'There are no fastq files for FastQCJob'):
             wf.execute_pipeline()
 
     def test_metatranscriptomic_workflow_creation(self):
@@ -147,7 +148,7 @@ def test_metatranscriptomic_workflow_creation(self):
         self.assertEqual(wf.assay_type, ASSAY_NAME_METATRANSCRIPTOMIC)
 
         with self.assertRaisesRegex(
-                NotFoundError, '{"message": "Study not found"}'):
+                PipelineError, 'There are no fastq files for FastQCJob'):
             wf.execute_pipeline()
 
     def test_amplicon_workflow_creation(self):
@@ -193,7 +194,8 @@ def test_tellseq_workflow_creation(self):
         self.assertEqual(wf.assay_type, ASSAY_NAME_METAGENOMIC)
 
         with self.assertRaisesRegex(
-                NotFoundError, '{"message": "Study not found"}'):
+                FileNotFoundError,
+                'TellReadJob/sample_index_list_TellReadJob.txt'):
             wf.execute_pipeline()