bug-fixes after testing on qiita-rc (#81)

* bug-fixes after testing on qiita-rc

* bugfix for error in CI

* updates

---------

Co-authored-by: charlie <[email protected]>

qp_klp/Step.py

@@ -916,7 +916,7 @@ def execute_pipeline(self, qclient, increment_status, update=True,
             self.convert_bcl_to_fastq()
 
         increment_status()
-        if "QCJob" not in skip_steps:
+        if "NuQCJob" not in skip_steps:
             self.quality_control()
 
         increment_status()

qp_klp/klp.py

@@ -79,17 +79,18 @@ def sequence_processing_pipeline(qclient, job_id, parameters, out_dir):
     # successful.
 
     # at minimum, ConvertJob needs to have been successful.
-    is_restart = True if exists(join(out_dir, 'NuQCJob')) else False
+    out_path = partial(join, out_dir)
+    is_restart = True if exists(out_path('NuQCJob')) else False
 
     if is_restart:
         # Assume ConvertJob directory exists and parse the job-script found
         # there. If this is a restart, we won't be given the run-identifier,
         # the lane number, and the sample-sheet as input parameters.
         some_path = join(out_dir, 'ConvertJob', 'ConvertJob.sh')
         result = ConvertJob.parse_job_script(some_path)
-        run_identifier = split(result['out_directory'])[-1]
-        user_input_file = result['sample_sheet_path']
-        sheet = load_sample_sheet(user_input_file)
+        run_identifier = split(result['run_directory'])[-1]
+        uif_path = result['sample_sheet_path']
+        sheet = load_sample_sheet(uif_path)
         # on Amplicon runs, lane_number is always 1, and this will be
         # properly reflected in the dummy sample-sheet as well.
         lane_number = sheet.get_lane_number()
@@ -113,23 +114,17 @@ def sequence_processing_pipeline(qclient, job_id, parameters, out_dir):
         user_input_file = parameters.pop('sample_sheet')
         lane_number = parameters.pop('lane_number')
 
-    if {'body', 'content_type', 'filename'} != set(user_input_file):
-        return False, None, ("This doesn't appear to be a valid sample sheet "
-                             "or mapping file; please review.")
-
-    out_path = partial(join, out_dir)
-    final_results_path = out_path('final_results')
-    makedirs(final_results_path, exist_ok=True)
-    # replace any whitespace in the filename with underscores
-    uif_path = out_path(user_input_file['filename'].replace(' ', '_'))
-
-    if is_restart:
-        pass
-    else:
+        if {'body', 'content_type', 'filename'} != set(user_input_file):
+            return False, None, ("This doesn't appear to be a valid sample "
+                                 "sheet or mapping file; please review.")
+        uif_path = out_path(user_input_file['filename'].replace(' ', '_'))
         # save raw data to file
         with open(uif_path, 'w') as f:
             f.write(user_input_file['body'])
 
+    final_results_path = out_path('final_results')
+    makedirs(final_results_path, exist_ok=True)
+
     if Pipeline.is_sample_sheet(uif_path):
         with open(uif_path, 'r') as f:
             assay = [x for x in f.readlines() if 'Assay' in x]

qp_klp/tests/data/process_all_fastq_files.sh

@@ -12,18 +12,13 @@
 
 echo "---------------"
 echo "Run details:"
-echo "$SLURM_JOB_NAME $SLURM_JOB_ID $SLURMD_NODENAME"
+echo "$SLURM_JOB_NAME $SLURM_JOB_ID $SLURMD_NODENAME $SLURM_ARRAY_TASK_ID"
 echo "---------------"
 
 if [[ -z "${SLURM_ARRAY_TASK_ID}" ]]; then
     echo "Not operating within an array"
     exit 1
 fi
-
-if [[ "${SLURM_ARRAY_TASK_MIN}" -ne 1 ]]; then
-    echo "Min array ID is not 1"
-    exit 1
-fi
 if [[ -z ${MMI} ]]; then
     echo "MMI is not set"
     exit 1
@@ -56,11 +51,7 @@ fi
 # DO NOT do this casually. Only do a clean up like this if
 # you know for sure TMPDIR is what you want.
 
-# we might got back to this TMPDIR once the slurm scheduler controls it
-#    TMPDIR=/dev/shm
-# right now, let's use ${OUTPUT}, note that each worker will create a new tmp
-# folder two lines below
-TMPDIR=${OUTPUT}
+TMPDIR=/dev/shm
 export TMPDIR=${TMPDIR}
 export TMPDIR=$(mktemp -d)
 echo $TMPDIR