Skip to content

Commit 09b18cf

Browse files
kenwenzelkenwenzel
committed
First sketch of MDB_DUPSORT.
1 parent 6918302 commit 09b18cf

File tree

14 files changed

+904
-1456
lines changed

14 files changed

+904
-1456
lines changed

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java‎

Lines changed: 87 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,19 @@
1111
package org.eclipse.rdf4j.sail.lmdb;
1212

1313
import static org.eclipse.rdf4j.sail.lmdb.LmdbUtil.E;
14+
import static org.lwjgl.util.lmdb.LMDB.MDB_GET_BOTH_RANGE;
1415
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT;
16+
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT_DUP;
17+
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT_NODUP;
1518
import static org.lwjgl.util.lmdb.LMDB.MDB_NOTFOUND;
16-
import static org.lwjgl.util.lmdb.LMDB.MDB_SET;
1719
import static org.lwjgl.util.lmdb.LMDB.MDB_SET_RANGE;
1820
import static org.lwjgl.util.lmdb.LMDB.MDB_SUCCESS;
1921
import static org.lwjgl.util.lmdb.LMDB.mdb_cmp;
2022
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_close;
2123
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_get;
2224
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_open;
2325
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_renew;
26+
import static org.lwjgl.util.lmdb.LMDB.mdb_dcmp;
2427

2528
import java.io.IOException;
2629
import java.nio.ByteBuffer;
@@ -29,7 +32,7 @@
2932
import org.eclipse.rdf4j.sail.SailException;
3033
import org.eclipse.rdf4j.sail.lmdb.TripleStore.TripleIndex;
3134
import org.eclipse.rdf4j.sail.lmdb.TxnManager.Txn;
32-
import org.eclipse.rdf4j.sail.lmdb.util.GroupMatcher;
35+
import org.eclipse.rdf4j.sail.lmdb.util.EntryMatcher;
3336
import org.lwjgl.PointerBuffer;
3437
import org.lwjgl.system.MemoryStack;
3538
import org.lwjgl.util.lmdb.MDBVal;
@@ -40,22 +43,19 @@
4043
* A record iterator that wraps a native LMDB iterator.
4144
*/
4245
class LmdbRecordIterator implements RecordIterator {
46+
4347
private static final Logger log = LoggerFactory.getLogger(LmdbRecordIterator.class);
4448
private final Pool pool;
4549

4650
private final TripleIndex index;
4751

48-
private final long subj;
49-
private final long pred;
50-
private final long obj;
51-
private final long context;
52-
5352
private final long cursor;
5453

5554
private final MDBVal maxKey;
55+
private final MDBVal maxValue;
5656

5757
private final boolean matchValues;
58-
private GroupMatcher groupMatcher;
58+
private EntryMatcher matcher;
5959

6060
private final Txn txnRef;
6161

@@ -73,44 +73,56 @@ class LmdbRecordIterator implements RecordIterator {
7373

7474
private ByteBuffer minKeyBuf;
7575

76-
private ByteBuffer maxKeyBuf;
76+
private ByteBuffer minValueBuf;
7777

78-
private int lastResult;
78+
private final ByteBuffer maxKeyBuf;
79+
80+
private final ByteBuffer maxValueBuf;
7981

8082
private final long[] quad;
81-
private final long[] originalQuad;
83+
private final long[] patternQuad;
8284

8385
private boolean fetchNext = false;
8486

8587
private final StampedLongAdderLockManager txnLockManager;
8688

8789
private final Thread ownerThread = Thread.currentThread();
8890

89-
LmdbRecordIterator(TripleIndex index, boolean rangeSearch, long subj, long pred, long obj,
91+
private final int indexScore;
92+
93+
LmdbRecordIterator(TripleIndex index, int indexScore, long subj, long pred, long obj,
9094
long context, boolean explicit, Txn txnRef) throws IOException {
91-
this.subj = subj;
92-
this.pred = pred;
93-
this.obj = obj;
94-
this.context = context;
95-
this.originalQuad = new long[] { subj, pred, obj, context };
95+
this.patternQuad = new long[] { subj, pred, obj, context };
9696
this.quad = new long[] { subj, pred, obj, context };
9797
this.pool = Pool.get();
9898
this.keyData = pool.getVal();
9999
this.valueData = pool.getVal();
100100
this.index = index;
101-
if (rangeSearch) {
101+
this.indexScore = indexScore;
102+
// prepare min and max keys if index can be used
103+
// otherwise, leave as null to indicate full scan
104+
if (indexScore > 0) {
102105
minKeyBuf = pool.getKeyBuffer();
103-
index.getMinKey(minKeyBuf, subj, pred, obj, context);
106+
minValueBuf = pool.getKeyBuffer();
107+
index.getMinEntry(minKeyBuf, minValueBuf, subj, pred, obj, context);
104108
minKeyBuf.flip();
109+
minValueBuf.flip();
105110

106111
this.maxKey = pool.getVal();
112+
this.maxValue = pool.getVal();
107113
this.maxKeyBuf = pool.getKeyBuffer();
108-
index.getMaxKey(maxKeyBuf, subj, pred, obj, context);
114+
this.maxValueBuf = pool.getKeyBuffer();
115+
index.getMaxEntry(maxKeyBuf, maxValueBuf, subj, pred, obj, context);
109116
maxKeyBuf.flip();
117+
maxValueBuf.flip();
110118
this.maxKey.mv_data(maxKeyBuf);
119+
this.maxValue.mv_data(maxValueBuf);
111120
} else {
112121
minKeyBuf = null;
113122
this.maxKey = null;
123+
this.maxValue = null;
124+
this.maxKeyBuf = null;
125+
this.maxValueBuf = null;
114126
}
115127

116128
this.matchValues = subj > 0 || pred > 0 || obj > 0 || context >= 0;
@@ -147,6 +159,7 @@ public long[] next() {
147159
} catch (InterruptedException e) {
148160
throw new SailException(e);
149161
}
162+
int lastResult;
150163
try {
151164
if (closed) {
152165
log.debug("Calling next() on an LmdbRecordIterator that is already closed, returning null");
@@ -161,15 +174,21 @@ public long[] next() {
161174
// cursor must be positioned on last item, reuse minKeyBuf if available
162175
if (minKeyBuf == null) {
163176
minKeyBuf = pool.getKeyBuffer();
177+
minValueBuf = pool.getKeyBuffer();
164178
}
165179
minKeyBuf.clear();
166-
index.toKey(minKeyBuf, quad[0], quad[1], quad[2], quad[3]);
180+
index.toEntry(minKeyBuf, minValueBuf, quad[0], quad[1], quad[2], quad[3]);
167181
minKeyBuf.flip();
182+
minValueBuf.flip();
168183
keyData.mv_data(minKeyBuf);
169-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET);
170-
if (lastResult != MDB_SUCCESS) {
171-
// use MDB_SET_RANGE if key was deleted
172-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
184+
// use set range if entry was deleted
185+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
186+
if (lastResult == MDB_SUCCESS) {
187+
valueData.mv_data(minValueBuf);
188+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
189+
if (lastResult != MDB_SUCCESS) {
190+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
191+
}
173192
}
174193
if (lastResult != MDB_SUCCESS) {
175194
closeInternal(false);
@@ -180,14 +199,30 @@ public long[] next() {
180199
this.txnRefVersion = txnRef.version();
181200
}
182201

202+
boolean isDupValue = false;
183203
if (fetchNext) {
184-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
204+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_DUP);
205+
if (lastResult != MDB_SUCCESS) {
206+
// no more duplicates, move to next key
207+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_NODUP);
208+
} else {
209+
isDupValue = true;
210+
}
185211
fetchNext = false;
186212
} else {
187213
if (minKeyBuf != null) {
188214
// set cursor to min key
189215
keyData.mv_data(minKeyBuf);
190-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
216+
// set range on key is only required if less than the first two key elements are fixed
217+
lastResult = indexScore >= 2 ? MDB_SUCCESS
218+
: mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
219+
if (lastResult == MDB_SUCCESS) {
220+
valueData.mv_data(minValueBuf);
221+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
222+
if (lastResult != MDB_SUCCESS) {
223+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
224+
}
225+
}
191226
} else {
192227
// set cursor to first item
193228
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
@@ -196,14 +231,22 @@ public long[] next() {
196231

197232
while (lastResult == MDB_SUCCESS) {
198233
// if (maxKey != null && TripleStore.COMPARATOR.compare(keyData.mv_data(), maxKey.mv_data()) > 0) {
199-
if (maxKey != null && mdb_cmp(txn, dbi, keyData, maxKey) > 0) {
234+
int keyDiff;
235+
if (maxKey != null &&
236+
(keyDiff = isDupValue ? 0 : mdb_cmp(txn, dbi, keyData, maxKey)) >= 0
237+
&& (keyDiff > 0 || mdb_dcmp(txn, dbi, valueData, maxValue) > 0)) {
200238
lastResult = MDB_NOTFOUND;
201-
} else if (matches()) {
239+
} else if (notMatches(isDupValue)) {
202240
// value doesn't match search key/mask, fetch next value
203-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
241+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_DUP);
242+
if (lastResult != MDB_SUCCESS) {
243+
// no more duplicates, move to next key
244+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_NODUP);
245+
}
246+
isDupValue = false;
204247
} else {
205248
// Matching value found
206-
index.keyToQuad(keyData.mv_data(), originalQuad, quad);
249+
index.entryToQuad(keyData.mv_data(), valueData.mv_data(), patternQuad, quad);
207250
// fetch next value
208251
fetchNext = true;
209252
return quad;
@@ -216,13 +259,17 @@ public long[] next() {
216259
}
217260
}
218261

219-
private boolean matches() {
220-
221-
if (groupMatcher != null) {
222-
return !this.groupMatcher.matches(keyData.mv_data());
262+
private boolean notMatches(boolean testValueOnly) {
263+
if (matcher != null) {
264+
if (testValueOnly) {
265+
// already positioned on correct key, no need to match key again
266+
return !this.matcher.matchesValue(valueData.mv_data());
267+
}
268+
return !this.matcher.matches(keyData.mv_data(), valueData.mv_data());
223269
} else if (matchValues) {
224-
this.groupMatcher = index.createMatcher(subj, pred, obj, context);
225-
return !this.groupMatcher.matches(keyData.mv_data());
270+
// lazy init of group matcher
271+
this.matcher = index.createMatcher(patternQuad[0], patternQuad[1], patternQuad[2], patternQuad[3]);
272+
return !this.matcher.matches(keyData.mv_data(), valueData.mv_data());
226273
} else {
227274
return false;
228275
}
@@ -247,10 +294,13 @@ private void closeInternal(boolean maybeCalledAsync) {
247294
pool.free(valueData);
248295
if (minKeyBuf != null) {
249296
pool.free(minKeyBuf);
297+
pool.free(minValueBuf);
250298
}
251299
if (maxKey != null) {
252300
pool.free(maxKeyBuf);
253301
pool.free(maxKey);
302+
pool.free(maxValueBuf);
303+
pool.free(maxValue);
254304
}
255305
}
256306
} finally {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/Pool.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ class Pool {
2222
// thread-local pool instance
2323
private static final ThreadLocal<Pool> threadlocal = ThreadLocal.withInitial(Pool::new);
2424

25-
private final MDBVal[] valPool = new MDBVal[1024];
26-
private final ByteBuffer[] keyPool = new ByteBuffer[1024];
25+
private final MDBVal[] valPool = new MDBVal[2048];
26+
private final ByteBuffer[] keyPool = new ByteBuffer[2048];
2727
private final Statistics[] statisticsPool = new Statistics[512];
2828
private int valPoolIndex = -1;
2929
private int keyPoolIndex = -1;

0 commit comments

Comments
 (0)