GH-3994: Improved TDB2 index access for single pattern queries

jena-db/jena-dboe-base/src/main/java/org/apache/jena/dboe/base/record/Record.java

@@ -167,4 +167,53 @@ static boolean compatible(Record record1, Record record2, boolean checkValue) {
         }
         return true;
     }
+
+    /**
+     * Return a new record with the key increased by 1.
+     * Returned record shares the value array of the argument record.
+     */
+    public static Record incrementKey(Record record) {
+        return new Record(incrementByteArray(record.getKey().clone()), record.getValue());
+    }
+
+    private static byte[] incrementByteArray(byte[] array) {
+        // Loop from the rightmost (least significant) byte to the left
+        for (int i = array.length - 1; i >= 0; i--) {
+            array[i]++;
+            // If it didn't roll over to 0, no carry is needed; we are done
+            if (array[i] != 0) {
+                return array;
+            }
+        }
+        // If we reach here, the entire array overflowed (e.g., [255, 255] -> [0, 0])
+        throw new ArithmeticException("Byte array overflowed its maximum value");
+    }
+
+    /**
+     * Pad {@code record} with zeros such that the lengths of its
+     * key and value arrays match those of the {@code prototype} record.
+     *
+     * If no padding is needed then {@code record} is returned.
+     * Otherwise, a new record is created.
+     */
+    public static Record padRight(Record record, Record prototype) {
+        byte[] rkey = record.getKey();
+        byte[] rval = record.getValue();
+
+        byte[] pkey = prototype.getKey();
+        byte[] pval = prototype.getValue();
+
+        byte[] key = pkey == null ? rkey : padRight(rkey, pkey.length);
+        byte[] val = pval == null ? rval : padRight(rval, pval.length);
+        return (key == rkey && val == rval) ? record : new Record(key, val);
+    }
+
+    private static byte[] padRight(byte[] arr, int length) {
+        if (length == arr.length) {
+            return arr;
+        } else if (length < arr.length) {
+            return Arrays.copyOf(arr, arr.length);
+        }
+        return Arrays.copyOf(arr, length);
+    }
 }

jena-db/jena-dboe-trans-data/src/main/java/org/apache/jena/dboe/trans/bplustree/BPTreeDistinctKeyPrefixIterator.java

@@ -57,10 +57,30 @@ public static Iterator<Record> create(BPTreeNode node, int keyPrefixLength) {
         return new BPTreeDistinctKeyPrefixIterator(node, keyPrefixLength, minRecord, maxRecord);
     }
 
+    /** Create an iterator of distinct keys over a custom sub-range. Arguments must not be null. */
+    public static Iterator<Record> create(BPTreeNode node, int keyPrefixLength, Record minRecord, Record maxRecord) {
+        Objects.requireNonNull(minRecord);
+        Objects.requireNonNull(maxRecord);
+
+        Record nodeMinRecord = node.minRecord();
+        Record nodeMaxRecord = node.maxRecord();
+        // If there's no records just return a null iterator
+        if (nodeMinRecord == null) {
+            return Iter.nullIter();
+        }
+
+        // Pad records to make their lengths match those of the nodes.
+        Record paddedMinRecord = Record.padRight(minRecord, nodeMinRecord);
+        Record paddedMaxRecord = Record.padRight(maxRecord, nodeMaxRecord);
+
+        return new BPTreeDistinctKeyPrefixIterator(node, keyPrefixLength, paddedMinRecord, paddedMaxRecord);
+    }
+
     // Convert path to a stack of iterators
     private final Deque<Iterator<BPTreePage>> stack = new ArrayDeque<>();
     private Iterator<Record> current;
-    private Record slot = null, minRecord, maxRecord;
+    private Record slot = null;
+    private final Record minRecord, maxRecord, maxRecordPlusOne;
     private byte[] lastPrefix = null;
     private boolean finished = false;
 
@@ -70,6 +90,7 @@ public static Iterator<Record> create(BPTreeNode node, int keyPrefixLength) {
         this.keyPrefixLength = keyPrefixLength;
         this.minRecord = minRecord;
         this.maxRecord = maxRecord;
+        this.maxRecordPlusOne = Record.incrementKey(maxRecord);
 
         BPTreeRecords r = loadStack(node);
         current = getRecordsIterator(r);
@@ -143,6 +164,12 @@ private Iterator<Record> moveOnCurrent() {
         return getRecordsIterator(r);
     }
 
+    protected final int comparePrefix(Record a, Record b) {
+        // Need to compare unsigned in general:
+        // E.g. [0, 5] is less than [0, -1] because the latter must be treated as [0, 255].
+        return Arrays.compareUnsigned(a.getKey(), 0, this.keyPrefixLength, b.getKey(), 0, this.keyPrefixLength);
+    }
+
     protected final boolean haveSamePrefix(Record a, Record b) {
         return Arrays.compare(a.getKey(), 0, this.keyPrefixLength, b.getKey(), 0, this.keyPrefixLength) == 0;
     }
@@ -157,12 +184,18 @@ protected Iterator<Record> getRecordsIterator(BPTreeRecords records) {
         } else {
             // Check whether we need to scan the whole page or can process it by skipping or singleton yield
             Record lowRecord = records.getLowRecord();
-            if (haveSamePrefix(lowRecord, records.getHighRecord())) {
-                // If the low and high keys for this page have the same prefix then just return a singleton iterator
-                iter = Iter.singletonIterator(lowRecord);
+            Record highRecord= records.getHighRecord();
+            if (haveSamePrefix(lowRecord, highRecord)) {
+                // Check that the prefix is in range
+                if (comparePrefix(lowRecord, minRecord) >= 0 && comparePrefix(highRecord, maxRecord) <= 0) {
+                    // If the low and high keys for this page have the same prefix then just return a singleton iterator
+                    iter = Iter.singletonIterator(lowRecord);
+                } else {
+                    iter = Iter.nullIterator();
+                }
             } else {
                 // Otherwise need to scan the whole page
-                iter = records.getRecordBuffer().iterator();
+                iter = records.getRecordBuffer().iterator(minRecord, maxRecordPlusOne);
             }
         }
         records.bpTree.finishReadBlkMgr();
@@ -173,7 +206,7 @@ private BPTreeRecords loadStack(BPTreeNode node) {
         AccessPath path = new AccessPath(null);
         node.bpTree.startReadBlkMgr();
 
-        node.internalMinRecord(path);
+        node.internalSearch(path, minRecord);
         List<AccessStep> steps = path.getPath();
         for (AccessStep step : steps) {
             BPTreeNode n = step.node;

jena-db/jena-dboe-trans-data/src/main/java/org/apache/jena/dboe/trans/bplustree/BPlusTree.java

@@ -329,6 +329,14 @@ public Iterator<Record> distinctByKeyPrefix(int keyPrefixLength) {
         return BPTreeDistinctKeyPrefixIterator.create(root, keyPrefixLength);
     }
 
+    public Iterator<Record> distinctByKeyPrefix(int keyPrefixLength, Record fromRecInclusive, Record toRecInclusive) {
+        startReadBlkMgr();
+        BPTreeNode root = getRootRead();
+        releaseRootRead(root);
+        finishReadBlkMgr();
+        return BPTreeDistinctKeyPrefixIterator.create(root, keyPrefixLength, fromRecInclusive, toRecInclusive);
+    }
+
     /*
     @Override
     public <X> Iterator<X> iterator(Record minRec, Record maxRec, RecordMapper<X> mapper) {

jena-geosparql/src/main/java/org/apache/jena/geosparql/spatial/task/BasicTask.java

@@ -56,7 +56,7 @@ public interface TaskListener<T extends BasicTask> {
     /**
      * Whether the task has terminated.
      *
-     * @implNote The original name {@code isTerminated} collided with the package−private method in {@link Thread}.
+     * @implNote The original name {@code isTerminated} collided with the package-private method in {@link Thread}.
      */
     default boolean hasTerminated() {
         TaskState state = getTaskState();

jena-tdb2/pom.xml

@@ -16,7 +16,7 @@
    KIND, either express or implied.  See the License for the
    specific language governing permissions and limitations
    under the License.
-    
+
      SPDX-License-Identifier: Apache-2.0
 -->
 
@@ -32,7 +32,7 @@
     <groupId>org.apache.jena</groupId>
     <artifactId>jena</artifactId>
     <version>6.3.0-SNAPSHOT</version>
-  </parent> 
+  </parent>
 
   <properties>
     <build.time.xsd>${maven.build.timestamp}</build.time.xsd>
@@ -73,6 +73,11 @@
       <scope>test</scope>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-math4-legacy</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/OpExecutorTDB2.java

@@ -47,9 +47,13 @@
 import org.apache.jena.sparql.engine.optimizer.reorder.ReorderTransformation;
 import org.apache.jena.sparql.expr.ExprList;
 import org.apache.jena.sparql.mgt.Explain;
+import org.apache.jena.tdb2.solver.skipscan.OpExecutorTDB2SkipScan;
+import org.apache.jena.tdb2.solver.skipscan.OpExtSkipScan;
+import org.apache.jena.tdb2.solver.skipscan.SkipScanRewrite;
 import org.apache.jena.tdb2.store.DatasetGraphTDB;
 import org.apache.jena.tdb2.store.GraphTDB;
 import org.apache.jena.tdb2.store.NodeId;
+import org.apache.jena.tdb2.sys.SystemTDB;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -92,13 +96,40 @@ protected QueryIterator exec(Op op, QueryIterator input) {
 
     // Retrieving nodes isn't so bad because they will be needed anyway.
     // And if their duplicates, likely to be cached.
-    // Need to work with SolverLib which wraps the NodeId bindgins with a converter.
+    // Need to work with SolverLib which wraps the NodeId bindings with a converter.
 
     @Override
     protected QueryIterator execute(OpDistinct opDistinct, QueryIterator input) {
+        if ( isForTDB ) {
+            boolean isSkipScanEnabled = execCxt.getContext().isTrueOrUndef(SystemTDB.symSkipScan);
+            if ( isSkipScanEnabled ) {
+                Op rewritten = SkipScanRewrite.tryRewriteAsSkipScan(opDistinct);
+                if (rewritten != null && !opDistinct.equals(rewritten)) {
+                    OpExtSkipScan.setOpExecutorIfAbsent(execCxt.getContext(), this);
+                    return exec(rewritten, input);
+                }
+            }
+        }
+
         return super.execute(opDistinct, input);
     }
 
+    @Override
+    protected QueryIterator execute(OpGroup opGroup, QueryIterator input) {
+        if ( isForTDB ) {
+            boolean isSkipScanEnabled = execCxt.getContext().isTrueOrUndef(SystemTDB.symSkipScan);
+            if ( isSkipScanEnabled ) {
+                Op rewritten = SkipScanRewrite.tryRewriteAsSkipScan(opGroup);
+                if (rewritten != null) {
+                    OpExtSkipScan.setOpExecutorIfAbsent(execCxt.getContext(), this);
+                    return exec(rewritten, input);
+                }
+            }
+        }
+
+        return super.execute(opGroup, input);
+    }
+
     @Override
     protected QueryIterator execute(OpReduced opReduced, QueryIterator input) {
         return super.execute(opReduced, input);

jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/skipscan/CandidateIndex.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *   SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.apache.jena.tdb2.solver.skipscan;
+
+import org.apache.jena.atlas.lib.tuple.Tuple;
+import org.apache.jena.tdb2.store.tupletable.TupleIndexRecord;
+
+record CandidateIndex<T>(
+    TupleIndexRecord index,
+    Tuple<T> pattern,
+    int[] projectIndices,
+    ValueFilter<T> residualConditions
+) {}

jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/skipscan/EqualityLink.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *   SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.apache.jena.tdb2.solver.skipscan;
+
+/**
+ * Declare that the value of a row at column index {@code equalToIdx} must be equal
+ * to the value at column index {@code idx}.
+ * This case occurs for queries with patterns such as (?s :p ?s):
+ * Here, column 2 and 0 are linked by an equality constraint.
+ */
+record EqualityLink(int idx, int equalToIdx) {}

jena-tdb2/src/main/java/org/apache/jena/tdb2/solver/skipscan/IndexMap.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ *   SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.apache.jena.tdb2.solver.skipscan;
+
+import java.util.Arrays;
+
+import org.apache.jena.graph.Node;
+import org.apache.jena.tdb2.store.NodeId;
+
+record IndexMap(
+    Node[] nodeTuple,
+    NodeId[] tuple,
+    VarMap[] proj,
+    ValueFilter<NodeId>[] residualConditions,
+    EqualityLink[] equalityLinks)
+{
+
+    @Override
+    public String toString() {
+        return "IndexMap [nodeTuple=" + Arrays.toString(nodeTuple) + ", tuple=" + Arrays.toString(tuple) + ", proj="
+                + Arrays.toString(proj) + ", residualConditions=" + Arrays.toString(residualConditions) + ", links="
+                + Arrays.toString(equalityLinks) + "]";
+    }
+}