add pipeline dir to organize execution

Author: DanHUMassMed

.gitignore

@@ -1,8 +1,7 @@
 # Directories to ignore
 .nextflow/
-data/
-results/
-results-backups/
+**/data/
+**/results/
 work/
 
 # Files to ignore

README.md

@@ -27,7 +27,7 @@ The pipeline takes FASTQ files as input, performs initial MD5 checks, quality co
 * 5c. Produce heatmap visualizations  
 * 6a. Execute Wormcat Batch (`wormcat_batch.nf`)
 
-
+Note: when running use `nextflow run PIPLINE.nf -bg -N [email protected]`, which will run in the background and email when the process terminates (success or failure)
 
 ## Pipeline Outputs
 

bin/expression_summary.py

@@ -16,18 +16,20 @@ def extract_salmon_experiment_name(file_path):
     experiment_name = directory_path[index:]
     return experiment_name
 
-def find_files_with_suffix(directory, suffix):
+def find_files_with_suffix(directory, dir_prefix, file_suffix):
     matching_files = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith(suffix):
-                matching_files.append(os.path.join(root, file))
+    for root, dirs, files in os.walk(directory,followlinks=True):
+        if dir_prefix in root:
+            print(f"{root=}")
+            for file in files:
+                if file.endswith(file_suffix):
+                    matching_files.append(os.path.join(root, file))
     return matching_files
 
 def aggregate_expression_counts(input_path, execution_variables):
     experiment_data_dfs = []
 
-    results_files = find_files_with_suffix(input_path, execution_variables['file_suffix'])
+    results_files = find_files_with_suffix(input_path, execution_variables['dir_prefix'], execution_variables['file_suffix'])
     # Read in all the individual results from RSEM
     for  result_file in results_files:
         df = pd.read_csv(result_file, delimiter='\t')
@@ -59,12 +61,14 @@ def main():
     cmd_line_msg = "expression_summary.py --expression-type [rsem | salmon] --input-path [<base_directory>]"
     execution_variables = {
         'rsem':{'output_file':"genes_expression_expected_count.tsv",
+                'dir_prefix':'rsem_expression_',
                 'file_suffix':'genes.results',
                 'columns_to_keep':['gene_id', 'expected_count'],
                 'expression_type' : 'rsem',
                 'extract_experiment_name': extract_rsem_experiment_name
                 },
         'salmon':{'output_file':"transcript_expression_counts.tsv",
+                'dir_prefix':'salmon_expression_',
                 'file_suffix':'quant.sf',
                 'columns_to_keep':['Name', 'NumReads'],
                 'expression_type' : 'salmon',

bin/get_dropbox_data.sh

@@ -20,7 +20,6 @@ length=${#directories[@]}
 # Reset the IFS to its default value (space)
 IFS=" "
 
-
 # Iterate over the directories and provide the time it takes to copy
 # also provide info on the number of files to copy and how many have been copies so far
 counter=0

create_salmon_index.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 process DECOY_TRANSCRIPTOME {
     container 'danhumassmed/samtools-bedtools:1.0.1'

get_dropbox_data-mike.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 process TX2GENE {
     tag "$transcriptome.simpleName"
@@ -34,41 +33,42 @@ process TXIMPORT_COUNTS {
     script:
     """
     mkdir -p salmon_summary
-    tx_import.R --input-path ${input_path} --output-path salmon_summary --tx2gene ${tx2gene} --counts-method ${count_method}
+    ${launchDir}/bin/tx_import.R --input-path ${input_path} --output-path salmon_summary --tx2gene ${tx2gene} --counts-method ${count_method}
     """
 }
 
 process GET_DROPBOX_DATA {
     container 'danhumassmed/de-seq-tools:1.0.1'
-    publishDir baseDir, mode:'move'
+    publishDir params.outdir, mode:'copy'
 
     input:
     val data_remote 
     val data_local
-    
-    output:
-    path "${data_local}"
-    val "${data_local}", emit: data_local_dir
 
     script:
     """
     mkdir -p "${data_local}"
-    get_dropbox_data.sh "${data_remote}" "${data_local}"
+    ${launchDir}/bin/get_dropbox_data.sh "${data_remote}" "${data_local}"
     """
+
+    output:
+    path "${data_local}"
+    
 }
 
 process CHECK_MD5 {
     container 'danhumassmed/de-seq-tools:1.0.1'
-    publishDir params.outdir, mode:'copy'
+    publishDir params.reportdir, mode:'copy'
 
     input:
-    val data_local
+    path data_local
 
-    output:
-    path "md5_report.html"
-
     script:
     """
-    check_md5.py "${baseDir}/${data_local}"
+    ${launchDir}/bin/check_md5.py "${data_local}"
     """
+
+    output:
+    path "md5_report.html"
+
 }

modules/bedtools/main.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 process FASTQC {
     tag "FASTQC on $sample_id"

modules/de-seq-tools/main.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 process MULTIQC {
     container 'danhumassmed/fastqc-multiqc:1.0.1'

modules/fastqc/main.nf

@@ -81,7 +81,7 @@ process RSEM_SUMMARY {
     publishDir params.outdir, mode:'copy'
 
     input:
-    path('*')
+    path('rsem_expression_*')
 
     output:
     path "rsem_summary" 
@@ -90,6 +90,6 @@ process RSEM_SUMMARY {
     """
     mkdir -p rsem_summary
     cd rsem_summary
-    expression_summary.py  --expression-type rsem --input-path "${baseDir}/${params.outdir}"
+    ${launchDir}/bin/expression_summary.py  --expression-type rsem --input-path ..
     """
 }

modules/multiqc/main.nf

@@ -26,7 +26,7 @@ process SALMON_QUANTIFY_SINGLE  {
     path reads 
 
     output:
-    path "./salmon_expression_${reads.getName().split("\\.")[0]}"
+    path "salmon_expression_${reads.getName().split("\\.")[0]}"
 
     script:
     """
@@ -44,7 +44,7 @@ process SALMON_QUANTIFY{
     tuple val(pair_id), path(reads) 
 
     output:
-    path "./salmon_expression_${pair_id}"
+    path "salmon_expression_${pair_id}"
 
     script:
     """

modules/rsem/main.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 process STAR_INDEX {
     container "danhumassmed/star-rsem:1.0.1"

modules/salmon/main.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 // import modules
 include { DECOY_TRANSCRIPTOME } from '../bedtools'

modules/star/main.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 // import modules
 include { STAR_INDEX } from '../star'

modules/sub-workflow/index-salmon.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 include { FASTQC; FASTQC_SINGLE } from '../fastqc'
 include { SALMON_QUANTIFY; SALMON_QUANTIFY_SINGLE; SALMON_SUMMARY } from '../salmon'

modules/sub-workflow/index-star-rsem.nf

@@ -1,5 +1,3 @@
-params.outdir = 'results'
-
 
 include { FASTQC; FASTQC_SINGLE } from '../fastqc'
 include { STAR_ALIGN; STAR_ALIGN_SINGLE } from '../star'

modules/sub-workflow/rnaseq-salmon.nf

@@ -1,4 +1,3 @@
-params.outdir = 'results'
 
 //http://www.usadellab.org/cms/uploads/supplementary/Trimmomatic/TrimmomaticManual_V0.32.pdf
 

modules/sub-workflow/rnaseq-star-rsem.nf

@@ -0,0 +1,49 @@
+#!/usr/bin/env nextflow 
+
+/* 
+ * enables modules 
+ */
+nextflow.enable.dsl = 2
+
+/*
+ * RNA SEQ Pipeline optimized for Alex Byrne 
+ */
+
+// rclone lsd remote:"Francis lab_KB_wholeworm RNAseq data_March 2023_Share AW lab"
+// rclone lsd remote:"SamLiu_ Francis lab September 2023"
+
+//params.data_remote="Francis lab_KB_wholeworm RNAseq data_March 2023_Share AW lab/August 2023 experiment"
+//params.data_remote="Francis lab_KB_wholeworm RNAseq data_March 2023_Share AW lab/March 2023 experiment"
+params.data_remote="SamLiu_ Francis lab September 2023"
+params.data_local="Experiment3"
+params.outdir = "${projectDir}/data"
+params.reportdir = "${params.outdir}/${params.data_local}"
+
+
+
+log.info """\
+ R N A S E Q - N F   P I P E L I N E
+ ===================================
+ data_remote    : ${params.data_remote}
+ outdir         : ${params.outdir}
+ project_dir    : ${projectDir}
+ launch_dir     : ${launchDir}
+ """
+
+/* 
+ * main script flow
+ */
+
+include { GET_DROPBOX_DATA } from "${launchDir}/modules/de-seq-tools"
+include { CHECK_MD5 } from "${launchDir}/modules/de-seq-tools"
+
+workflow {
+  GET_DROPBOX_DATA(params.data_remote, params.data_local)
+  //CHECK_MD5(GET_DROPBOX_DATA.out.collect())
+}
+
+workflow.onComplete {
+	log.info ( workflow.success ? "\nDone! The data is avialable --> ${params.reportdir}\n" : "Oops .. something went wrong" )
+}
+
+

modules/trimmomatic/main.nf

@@ -11,11 +11,12 @@ nextflow.enable.dsl = 2
  * RNA SEQ Pipeline optimized for Alex Byrne 
  */
 
-//params.reads = "${baseDir}/data/mike_francis/**/*_{1,2}.fq.gz"
-params.reads = "${baseDir}/results/trimmed_sliding_window/**/*_{1,2}.fq.gz"
-params.star_index_dir="${baseDir}/results/star_index"
-params.rsem_reference_dir = "${baseDir}/results/rsem_index"
-params.outdir = "results"
+//params.reads = "${projectDir}/data/mike_francis/**/*_{1,2}.fq.gz"
+params.reads = "${projectDir}/data/Experiment3/**/*_{1,2}.fq.gz"
+params.star_index_dir="${launchDir}/pipelines/shared/results/star_index"
+params.rsem_reference_dir = "${launchDir}/pipelines/shared/results/rsem_index"
+params.outdir = "${projectDir}/results"
+
 
 log.info """\
  R N A S E Q - N F   P I P E L I N E
@@ -24,12 +25,13 @@ log.info """\
  star_index_dir     : ${params.star_index_dir}
  rsem_reference_dir : ${params.rsem_reference_dir}
  outdir             : ${params.outdir}
- base_dir           : ${baseDir}
+ project_dir        : ${projectDir}
+ launch_dir         : ${launchDir}
  """
 
 // import modules
-include { RNASEQ_STAR_RSEM } from './modules/sub-workflow/rnaseq-star-rsem'
-include { MULTIQC } from './modules/multiqc'
+include { RNASEQ_STAR_RSEM } from "${launchDir}/modules/sub-workflow/rnaseq-star-rsem"
+include { MULTIQC } from "${launchDir}/modules/multiqc"
 
 /* 
  * main script flow
@@ -42,5 +44,5 @@ workflow {
 }
 
 workflow.onComplete {
-	log.info ( workflow.success ? "\nDone! Open the following report in your browser --> ${baseDir}/$params.outdir/multiqc_rsem_report.html\n" : "Oops .. something went wrong" )
+	log.info ( workflow.success ? "\nDone! Open the following report in your browser --> ${params.outdir}/multiqc_rsem_report.html\n" : "Oops .. something went wrong" )
 }