A run is not complete until the BCL files appear, regardless of RTAComplete.txt

appearing. This may or may not resolve the pipeline failures we've had.

Author: tbooth

RunStatus.py

@@ -39,6 +39,9 @@ def __init__( self , run_folder , opts = '' ):
                 for r, l in sorted(self.runinfo_xml.read_and_length.items()):
                     self.trigger_cycles.append(self.trigger_cycles[-1] + int(l))
 
+                # Correct the last one
+                self.trigger_cycles[-1] -= 1
+
                 # We can do the well dups check after 70 cycles but we can only
                 # check the indexes after the last index read is complete, so wait for that
                 try:
@@ -68,7 +71,13 @@ def _is_sequencing_finished( self ):
         # however there were no runs where the RTAComplete.txt was not the last file written to the run folder.
         # So will only check for this file to determine if sequencing has finished or not
         RTACOMPLETE_LOCATION = os.path.join( self.run_path_folder , 'RTAComplete.txt' )
-        return os.path.exists( RTACOMPLETE_LOCATION )
+
+        # Oh but it's no longer that simple. NovaSeq has started writing this file before all the CBCL
+        # files are in place.
+        if not os.path.exists( RTACOMPLETE_LOCATION ):
+            return False
+
+        return self._is_read_finished(-1)
 
     def _exists( self, glob_pattern ):
         """ Returns if a file exists and caches the result.
@@ -91,7 +100,7 @@ def _is_read_finished( self, readnum ):
             # readnum counts from 1 so trigger_cycles[readnum] is the first cycle of the
             # next read (trigger_cycles[0] is always 1)
             cycle = self.trigger_cycles[int(readnum)]
-            return self._exists( "Data/Intensities/BaseCalls/L001/C{}.1/*".format(cycle) )
+            return self._exists( f"Data/Intensities/BaseCalls/L001/C{cycle}.1/*" )
         except Exception:
             return False
 
@@ -174,7 +183,7 @@ def get_machine_status( self ):
             return "complete"
         for n in range(len(self.trigger_cycles), 0 , -1):
             if self._is_read_finished(n):
-                return "read{}_complete".format(n)
+                return f"read{n}_complete"
         return "waiting_for_data"
 
 

doc/CopyComplete.txt

@@ -0,0 +1,24 @@
+Soooo...
+
+Up until now, one of the easiest things in the Illuminatus pipeline was deciding if
+a run was ready to demultiplex. If RTAComplete.txt is there, then we go.
+
+But recently a couple of runs have failed, because a CBCL file was missing.
+
+So it seems that wating for RTAComplete.txt is not enough. In NovaSeq there is another
+file, CopyComplete.txt. But this comes in an hour after RTAComplete.txt, and it seems
+that we were only missing the CBCL file by a few minutes.
+
+11:10 240722_A00291_0576_AHVHL5DMXY/RTAComplete.txt
+11:10 240722_A00291_0576_AHVHL5DMXY/Data/Intensities/BaseCalls/L002/C222.1/L002_2.cbcl
+12:43 240722_A00291_0576_AHVHL5DMXY/CopyComplete.txt
+
+So, our choices are:
+
+1) Wait for CopyComplete.txt and add 90 minutes to processing time.
+2) Add a short delay (5 mins?) to after RTAComplete.txt appears (yuk!)
+3) Look explicitly for the CBCL file
+4) Something else??
+
+I can't see a decent option 4. I think we need to do option 3. Fortunately we already have
+code for Read1 ready detection.

test/run_tests.sh

@@ -12,7 +12,8 @@ if [ -n "$VIRTUAL_ENV" ] ; then
 elif [ -e _illuminatus_venv ] ; then
     echo "Running: source ./_illuminatus_venv/bin/activate"
     source ./_illuminatus_venv/bin/activate
-    if [ "$(which python3)" != "$(readlink -f _illuminatus_venv)/bin/python3" ] ; then
+    if [ "$(readlink -f "$(dirname "$(which python3)")")/python3" != \
+         "$(readlink -f _illuminatus_venv)/bin/python3" ] ; then
         echo "FAILED - python3 is $(which python3) not $(readlink -f _illuminatus_venv)/bin/python3"
         exit 1
     fi

test/seqdata_examples/160603_M01270_0196_000000000-AKGDE/Data/Intensities/BaseCalls/L001/C612.1/foo.bcl

@@ -113,6 +113,8 @@ def test_redo_on_early_fail( self ):
             read1 processing at all.
         """
         run_info = self.use_run('160726_K00166_0120_BHCVH2BBXX', copy=True)
+        self.md('Data/Intensities/BaseCalls/L001/C318.1')
+        self.touch('Data/Intensities/BaseCalls/L001/C318.1/foo.bcl')
 
         # So the run is failed but still waiting for data
         self.touch('pipeline/failed')
@@ -176,6 +178,8 @@ def test_messy_redo( self ):
         self.md('pipeline/output/seqdata/pipeline')
         self.touch('pipeline/read1.done')
         self.touch('RTAComplete.txt')
+        self.md('Data/Intensities/BaseCalls/L001/C318.1')
+        self.touch('Data/Intensities/BaseCalls/L001/C318.1/foo.bcl')
 
         def no_redo(status):
             """ Check the status AND also check
@@ -317,6 +321,9 @@ def yes_redo(status):
 
         # Now we finish the reads
         self.touch('RTAComplete.txt')
+        self.md('Data/Intensities/BaseCalls/L001/C318.1')
+        self.touch('Data/Intensities/BaseCalls/L001/C318.1/foo.bcl')
+
         no_redo('reads_finished')
         self.assertEqual(gy()['MachineStatus:'], 'complete')
 
@@ -408,8 +415,10 @@ def yes_redo(status):
 
         no_redo('read1_finished')
 
-        # Adding an RTAComplete.txt file should not change this status
+        # Adding an RTAComplete.txt file and final BCL should not change this status
         self.touch('RTAComplete.txt')
+        self.md('Data/Intensities/BaseCalls/L001/C68.1')
+        self.touch('Data/Intensities/BaseCalls/L001/C68.1/foo.bcl')
         no_redo('read1_finished')
 
         # Adding read1.started should push us to the state where ops will trigger in parallel