more cross-checking

Author: fasterthanlime

original/divsufsort.c

@@ -182,6 +182,7 @@ sort_typeBstar(const sauchar_t *T, saidx_t *SA,
     BSTAR_dump("post-rank");
 
     /* Construct the inverse suffix array of type B* suffixes using trsort. */
+    SA_dump(SA, "pre-tr");
     trsort(ISAb, SA, m, 1);
 
     SA_dump(SA, "post-tr");

original/trsort.c

@@ -690,23 +690,48 @@ trsort(saidx_t *ISA, saidx_t *SA, saidx_t n, saidx_t depth) {
     first = SA;
     skip = 0;
     unsorted = 0;
+    // PETER
     do {
-      if((t = *first) < 0) { first -= t; skip += t; }
+      if ((t = *first) < 0) {
+        crosscheck("t < 0");
+        first -= t;
+        skip += t;
+        crosscheck("first=%d skip=%d", first-SA, skip);
+      }
       else {
-        if(skip != 0) { *(first + skip) = skip; skip = 0; }
+        crosscheck("t >= 0");
+        if (skip != 0) {
+          crosscheck("SA[first=%d + skip=%d] = skip=%d", first-SA, skip, skip);
+          *(first + skip) = skip;
+          skip = 0;
+        }
         last = SA + ISA[t] + 1;
+        crosscheck("last=%d", last-SA);
         if(1 < (last - first)) {
+          crosscheck("1<(last-first)");
           budget.count = 0;
+          SA_dump(SA, "tr_introsort(A)")
+          crosscheck("call tr_introsort ISA=%d ISAd=%d first=%d last=%d", ISA-SA, ISAd-SA, first-SA, last-SA);
           tr_introsort(ISA, ISAd, SA, first, last, &budget);
-          if(budget.count != 0) { unsorted += budget.count; }
-          else { skip = first - last; }
+          SA_dump(SA, "tr_introsort(B)")
+          if (budget.count != 0) {
+            unsorted += budget.count;
+          } else {
+            skip = first - last;
+          }
         } else if((last - first) == 1) {
           skip = -1;
         }
         first = last;
       }
     } while(first < (SA + n));
-    if(skip != 0) { *(first + skip) = skip; }
-    if(unsorted == 0) { break; }
+    if(skip != 0) {
+      crosscheck("skip != 0, trsort-end");
+      *(first + skip) = skip;
+    }
+    if(unsorted == 0) {
+      crosscheck("unsorted == 0, trsort-end");
+      break;
+    }
   }
 }

src/divsufsort.rs

@@ -275,6 +275,7 @@ fn sort_typeBstar(T: &Text, SA: &mut SuffixArray) -> SortTypeBstarResult {
         BSTAR_dump(&mut B, "post-rank");
 
         // Construct the inverse suffix array of type B* suffixes using trsort.
+        SA_dump(SA, "pre-tr");
         trsort::trsort(ISAb, SA, m, 1);
 
         SA_dump(SA, "post-tr");

src/trsort.rs

@@ -1125,22 +1125,36 @@ pub fn trsort(ISA: SAPtr, SA: &mut SuffixArray, n: Idx, depth: Idx) {
         skip = 0;
         unsorted = 0;
 
-        // do..while
+        // PETER
         loop {
-            // body for do..while
             t = SA[first];
             if (t < 0) {
+                crosscheck!("t < 0");
                 first -= t;
                 skip += t;
+                crosscheck!("first={} skip={}", first, skip);
             } else {
+                crosscheck!("t >= 0");
                 if (skip != 0) {
+                    crosscheck!("SA[first={} + skip={}] = skip={}", first, skip, skip);
                     SA[first + skip] = skip;
                     skip = 0;
                 }
                 last = SAPtr(SA[ISA + t] + 1);
+                crosscheck!("last={}", last);
                 if (1 < (last - first)) {
+                    crosscheck!("1<(last-first)");
                     budget.count = 0;
+                    SA_dump(&SA.range_to(..n), "tr_introsort(A)");
+                    crosscheck!(
+                        "call tr_introsort ISA={} ISAd={} first={} last={}",
+                        ISA,
+                        ISAd,
+                        first,
+                        last
+                    );
                     tr_introsort(ISA, ISAd, SA, first, last, &mut budget);
+                    SA_dump(&SA.range_to(..n), "tr_introsort(B)");
                     if (budget.count != 0) {
                         unsorted += budget.count;
                     } else {
@@ -1159,9 +1173,11 @@ pub fn trsort(ISA: SAPtr, SA: &mut SuffixArray, n: Idx, depth: Idx) {
         }
 
         if (skip != 0) {
+            crosscheck!("skip != 0, trsort-end");
             SA[first + skip] = skip;
         }
         if (unsorted == 0) {
+            crosscheck!("unsorted == 0, trsort-end");
             break;
         }
 

testdata/input.txt

@@ -3,3 +3,5 @@ A corpus may contain texts in a single language (monolingual corpus) or text dat
 Multilingual corpora that have been specially formatted for side-by-side comparison are called aligned parallel corpora. There are two main types of parallel corpora which contain texts in two languages. In a translation corpus, the texts in one language are translations of texts in the other language. In a comparable corpus, the texts are of the same kind and cover the same content, but they are not translations of each other.[1] To exploit a parallel text, some kind of text alignment identifying equivalent text segments (phrases or sentences) is a prerequisite for analysis. Machine translation algorithms for translating between two languages are often trained using parallel fragments comprising a first language corpus and a second language corpus which is an element-for-element translation of the first language corpus.[2]
 
 In order to make the corpora more useful for doing linguistic research, they are often subjected to a process known as annotation. An example of annotating a corpus is part-of-speech tagging, or POS-tagging, in which information about each word's part of speech (verb, noun, adjective, etc.) is added to the corpus in the form of tags. Another example is indicating the lemma (base) form of each word. When the language of the corpus is not a working language of the researchers who use it, interlinear glossing is used to make the annotation bilingual.
+
+Some corpora have further structured levels of analysis applied. In particular, a number of smaller corpora may be fully parsed. Such corpora are usually called Treebanks or Parsed Corpora. The difficulty of ensuring that the entire corpus is completely and consistently annotated means that these corpora are usually smaller, containing around one to three million words.