fixing some more tests

Author: antgonza

src/sequence_processing_pipeline/multiqc-bclconvert-config.yaml

@@ -8,9 +8,7 @@
         "nprocs": 16,
         "queue": "qiita",
         "wallclock_time_in_minutes": 216,
-        "modules_to_load": [
-          "bclconvert_3.7.5"
-        ],
+        "modules_to_load": ["bclconvert_3.7.5"],
         "executable_path": "bcl-convert",
         "per_process_memory_limit": "10gb"
       },
@@ -19,9 +17,7 @@
         "nprocs": 62,
         "queue": "qiita",
         "wallclock_time_in_minutes": 1022,
-        "modules_to_load": [
-          "bcl2fastq_2.20.0.222"
-        ],
+        "modules_to_load": ["bcl2fastq_2.20.0.222"],
         "executable_path": "bcl2fastq",
         "per_process_memory_limit": "100gb"
       },
@@ -30,12 +26,11 @@
         "cpus_per_task": 32,
         "queue": "qiita",
         "wallclock_time_in_minutes": 2028,
-        "minimap2_databases": ["/databases/minimap2/db_1.mmi", "/databases/minimap2/db_2.mmi"],
-        "modules_to_load": [
-          "fastp_0.20.1",
-          "samtools_1.12",
-          "minimap2_2.18"
+        "minimap2_databases": [
+          "/databases/minimap2/db_1.mmi",
+          "/databases/minimap2/db_2.mmi"
         ],
+        "modules_to_load": ["fastp_0.20.1", "samtools_1.12", "minimap2_2.18"],
         "fastp_executable_path": "fastp",
         "minimap2_executable_path": "minimap2",
         "samtools_executable_path": "samtools",
@@ -60,12 +55,10 @@
         "queue": "qiita",
         "nthreads": 62,
         "wallclock_time_in_minutes": 220,
-        "modules_to_load": [
-          "fastqc_0.11.5"
-        ],
+        "modules_to_load": ["fastqc_0.11.5"],
         "fastqc_executable_path": "fastqc",
         "multiqc_executable_path": "multiqc",
-        "multiqc_config_file_path": "sequence_processing_pipeline/multiqc-bclconvert-config.yaml",
+        "multiqc_config_file_path": "tests/data/multiqc-bclconvert-config.yaml",
         "job_total_memory_limit": "20gb",
         "job_pool_size": 120,
         "job_max_array_length": 2000

src/sequence_processing_pipeline/scripts/fake_squeue.py renamed to tests/bin/fake_squeue.py

@@ -8,9 +8,7 @@
         "nprocs": 64,
         "queue": "qiita",
         "wallclock_time_in_minutes": 1024,
-        "modules_to_load": [
-          "bcl2fastq_2.20.0.422"
-        ],
+        "modules_to_load": ["bcl2fastq_2.20.0.422"],
         "executable_path": "bcl2fastq",
         "per_process_memory_limit": "100gb"
       },
@@ -19,12 +17,11 @@
         "cpus_per_task": 32,
         "queue": "qiita",
         "wallclock_time_in_minutes": 2048,
-        "minimap2_databases": ["/databases/minimap2/db_1.mmi", "/databases/minimap2/db_2.mmi"],
-        "modules_to_load": [
-          "fastp_0.20.1",
-          "samtools_1.12",
-          "minimap2_2.18"
+        "minimap2_databases": [
+          "/databases/minimap2/db_1.mmi",
+          "/databases/minimap2/db_2.mmi"
         ],
+        "modules_to_load": ["fastp_0.20.1", "samtools_1.12", "minimap2_2.18"],
         "fastp_executable_path": "fastp",
         "minimap2_executable_path": "minimap2",
         "samtools_executable_path": "samtools",
@@ -48,12 +45,10 @@
         "queue": "qiita",
         "nthreads": 64,
         "wallclock_time_in_minutes": 240,
-        "modules_to_load": [
-          "fastqc_0.11.5"
-        ],
+        "modules_to_load": ["fastqc_0.11.5"],
         "fastqc_executable_path": "fastqc",
         "multiqc_executable_path": "multiqc",
-        "multiqc_config_file_path": "sequence_processing_pipeline/multiqc-bclconvert-config.yaml",
+        "multiqc_config_file_path": "tests/data/multiqc-bclconvert-config.yaml",
         "job_total_memory_limit": "20gb",
         "job_pool_size": 120,
         "job_max_array_length": 4000

tests/data/configuration_profiles/miseq_metatranscriptomic.json

@@ -27,9 +27,7 @@
         "nprocs": 16,
         "queue": "qiita",
         "wallclock_time_in_minutes": 216,
-        "modules_to_load": [
-          "bclconvert_3.7.5"
-        ],
+        "modules_to_load": ["bclconvert_3.7.5"],
         "executable_path": "bcl-convert",
         "per_process_memory_limit": "10gb"
       },
@@ -38,9 +36,7 @@
         "nprocs": 64,
         "queue": "qiita",
         "wallclock_time_in_minutes": 1024,
-        "modules_to_load": [
-          "bcl2fastq_2.20.0.422"
-        ],
+        "modules_to_load": ["bcl2fastq_2.20.0.422"],
         "executable_path": "bcl2fastq",
         "per_process_memory_limit": "100gb"
       },
@@ -49,12 +45,11 @@
         "cpus_per_task": 32,
         "queue": "qiita",
         "wallclock_time_in_minutes": 2048,
-        "minimap2_databases": ["/databases/minimap2/db_1.mmi", "/databases/minimap2/db_2.mmi"],
-        "modules_to_load": [
-          "fastp_0.20.1",
-          "samtools_1.12",
-          "minimap2_2.18"
+        "minimap2_databases": [
+          "/databases/minimap2/db_1.mmi",
+          "/databases/minimap2/db_2.mmi"
         ],
+        "modules_to_load": ["fastp_0.20.1", "samtools_1.12", "minimap2_2.18"],
         "fastp_executable_path": "fastp",
         "minimap2_executable_path": "minimap2",
         "samtools_executable_path": "samtools",
@@ -78,12 +73,10 @@
         "queue": "qiita",
         "nthreads": 64,
         "wallclock_time_in_minutes": 240,
-        "modules_to_load": [
-          "fastqc_0.11.5"
-        ],
+        "modules_to_load": ["fastqc_0.11.5"],
         "fastqc_executable_path": "fastqc",
         "multiqc_executable_path": "multiqc",
-        "multiqc_config_file_path": "sequence_processing_pipeline/multiqc-bclconvert-config.yaml",
+        "multiqc_config_file_path": "tests/data/multiqc-bclconvert-config.yaml",
         "job_total_memory_limit": "20gb",
         "job_pool_size": 120,
         "job_max_array_length": 4000

tests/data/configuration_profiles/novaseq_amplicon.json

@@ -8,9 +8,7 @@
         "nprocs": 16,
         "queue": "qiita",
         "wallclock_time_in_minutes": 216,
-        "modules_to_load": [
-          "bcl2fastq_2.20.0.422"
-        ],
+        "modules_to_load": ["bcl2fastq_2.20.0.422"],
         "executable_path": "bcl2fastq",
         "per_process_memory_limit": "10gb"
       },
@@ -19,9 +17,7 @@
         "nprocs": 16,
         "queue": "qiita",
         "wallclock_time_in_minutes": 216,
-        "modules_to_load": [
-          "bclconvert_3.7.5"
-        ],
+        "modules_to_load": ["bclconvert_3.7.5"],
         "executable_path": "bcl-convert",
         "per_process_memory_limit": "10gb"
       },
@@ -30,15 +26,9 @@
         "nprocs": 16,
         "queue": "qiita",
         "wallclock_time_in_minutes": 60,
-        "minimap2_databases": [
-          "/databases/minimap2/human-phix-db.mmi"
-        ],
+        "minimap2_databases": ["/databases/minimap2/human-phix-db.mmi"],
         "kraken2_database": "/databases/minimap2/hp_kraken-db.mmi",
-        "modules_to_load": [
-          "fastp_0.20.1",
-          "samtools_1.12",
-          "minimap2_2.18"
-        ],
+        "modules_to_load": ["fastp_0.20.1", "samtools_1.12", "minimap2_2.18"],
         "fastp_executable_path": "fastp",
         "minimap2_executable_path": "minimap2",
         "samtools_executable_path": "samtools",
@@ -51,12 +41,11 @@
         "cpus_per_task": 8,
         "queue": "qiita",
         "wallclock_time_in_minutes": 240,
-        "minimap2_databases": ["/databases/minimap2/db_1.mmi", "/databases/minimap2/db_2.mmi"],
-        "modules_to_load": [
-          "fastp_0.20.1",
-          "samtools_1.12",
-          "minimap2_2.18"
+        "minimap2_databases": [
+          "/databases/minimap2/db_1.mmi",
+          "/databases/minimap2/db_2.mmi"
         ],
+        "modules_to_load": ["fastp_0.20.1", "samtools_1.12", "minimap2_2.18"],
         "fastp_executable_path": "fastp",
         "minimap2_executable_path": "minimap2",
         "samtools_executable_path": "samtools",
@@ -80,12 +69,10 @@
         "queue": "qiita",
         "nthreads": 16,
         "wallclock_time_in_minutes": 60,
-        "modules_to_load": [
-          "fastqc_0.11.5"
-        ],
+        "modules_to_load": ["fastqc_0.11.5"],
         "fastqc_executable_path": "fastqc",
         "multiqc_executable_path": "multiqc",
-        "multiqc_config_file_path": "sequence_processing_pipeline/multiqc-bclconvert-config.yaml",
+        "multiqc_config_file_path": "tests/data/multiqc-bclconvert-config.yaml",
         "job_total_memory_limit": "20gb",
         "job_pool_size": 30,
         "job_max_array_length": 1000

tests/data/configuration_profiles/novaseq_metagenomic.json

@@ -1 +1,87 @@
-This is a file.
+title: "Sequence processing summaries"
+output_fn_name: "index.html"
+show_analysis_paths: False
+show_analysis_time: False
+
+run_modules:
+    - bclconvert
+    - fastqc
+    - fastp
+
+# including BLANKs in reports is now default operation for the pipeline
+sample_names_ignore:
+#     - "blank*"
+#     - "BLANK*"
+
+# This can optionally be set to True. When version-checking is enabled,
+# a message will appear in stdout as well as in the log-file. If these are not
+# reviewed then the effort will be wasted. Note FastQCJob calls multiqc
+# multiple times, once for each project specified in the sample-sheet.
+no_version_check: False
+
+# How to plot graphs. Different templates can override these settings, but
+# the default template can use interactive plots (Javascript using HighCharts)
+# or flat plots (images, using MatPlotLib). With interactive plots, the report
+# can prevent automatically rendering all graphs if there are lots of samples
+# to prevent the browser being locked up when the report opens.
+
+# Try to use only interactive javascript graphs
+plots_force_interactive: True
+# If interactive, don't plot on load if > this number of datasets
+num_datasets_plot_limit: 1
+# Swap tables for a beeswarm plot above this
+max_table_rows: 10000
+
+# Overwrite the defaults of which table columns are visible by default
+table_columns_visible:
+    # "Base Calling": False # total: True
+    "Sequence Quality (bclconvert raw)":
+        percent_fails: False
+        percent_duplicates: False
+        percent_gc: False
+        avg_sequence_length: True
+        total_sequences: True
+    Trimming:
+        #  R1 and R2 being stored in the same file
+        input_format: False
+        avg_sequence_length: False
+        total_record_count:  False
+        mean_sequence_length:    False
+        fraction_bp_trimmed: False
+        fraction_records_with_adapters: False
+    "Sequence Quality (trimmed)":
+        percent_fails: False
+        percent_duplicates: False
+        percent_gc: False
+        avg_sequence_length: True
+        total_sequences: True
+    "Human Filtering":
+        overall_alignment_rate: True
+    "Sequence Quality (filtered)":
+        percent_fails: False
+        percent_duplicates: False
+        percent_gc: False
+        avg_sequence_length: True
+        total_sequences: True
+
+module_order:
+    - bclconvert:
+         name: 'Base Calling'
+         # TODO: make sure we can split Stats.json files on a per-project basis
+         info: 'Conversion from BCL files to FASTQ files. Note this section currently includes samples generated in other projects, this is a known bug that will be addressed in the near future. '
+    - fastqc:
+        name: 'Sequence Quality (raw)'
+        info: 'Sequence quality and summary statistics for raw sequences.'
+        path_filters:
+            - '*fastqc.zip'
+            - '*.csv'
+    - fastp:
+        name: 'Sequence Quality (adapter trimmed)'
+        info: 'Summary statistics from adapter trimming and quality control with fastp.'
+        fn: '*.json'
+    - fastqc:
+        name: 'Sequence Quality (trimmed)'
+        info: 'Sequence quality and summary statistics after quality-control and adapter trimming.'
+        path_filters:
+          - '*trimmed_fastqc.zip'
+          - '*fastp_fastqc.zip'