Skip to content

Commit 7f85dc6

Browse files
kenwenzelkenwenzel
committed
First sketch of MDB_DUPSORT.
1 parent 6918302 commit 7f85dc6

File tree

14 files changed

+907
-1456
lines changed

14 files changed

+907
-1456
lines changed

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/LmdbRecordIterator.java‎

Lines changed: 90 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -11,16 +11,20 @@
1111
package org.eclipse.rdf4j.sail.lmdb;
1212

1313
import static org.eclipse.rdf4j.sail.lmdb.LmdbUtil.E;
14+
import static org.lwjgl.util.lmdb.LMDB.MDB_FIRST_DUP;
15+
import static org.lwjgl.util.lmdb.LMDB.MDB_GET_BOTH_RANGE;
1416
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT;
17+
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT_DUP;
18+
import static org.lwjgl.util.lmdb.LMDB.MDB_NEXT_NODUP;
1519
import static org.lwjgl.util.lmdb.LMDB.MDB_NOTFOUND;
16-
import static org.lwjgl.util.lmdb.LMDB.MDB_SET;
1720
import static org.lwjgl.util.lmdb.LMDB.MDB_SET_RANGE;
1821
import static org.lwjgl.util.lmdb.LMDB.MDB_SUCCESS;
1922
import static org.lwjgl.util.lmdb.LMDB.mdb_cmp;
2023
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_close;
2124
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_get;
2225
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_open;
2326
import static org.lwjgl.util.lmdb.LMDB.mdb_cursor_renew;
27+
import static org.lwjgl.util.lmdb.LMDB.mdb_dcmp;
2428

2529
import java.io.IOException;
2630
import java.nio.ByteBuffer;
@@ -29,7 +33,7 @@
2933
import org.eclipse.rdf4j.sail.SailException;
3034
import org.eclipse.rdf4j.sail.lmdb.TripleStore.TripleIndex;
3135
import org.eclipse.rdf4j.sail.lmdb.TxnManager.Txn;
32-
import org.eclipse.rdf4j.sail.lmdb.util.GroupMatcher;
36+
import org.eclipse.rdf4j.sail.lmdb.util.EntryMatcher;
3337
import org.lwjgl.PointerBuffer;
3438
import org.lwjgl.system.MemoryStack;
3539
import org.lwjgl.util.lmdb.MDBVal;
@@ -40,22 +44,19 @@
4044
* A record iterator that wraps a native LMDB iterator.
4145
*/
4246
class LmdbRecordIterator implements RecordIterator {
47+
4348
private static final Logger log = LoggerFactory.getLogger(LmdbRecordIterator.class);
4449
private final Pool pool;
4550

4651
private final TripleIndex index;
4752

48-
private final long subj;
49-
private final long pred;
50-
private final long obj;
51-
private final long context;
52-
5353
private final long cursor;
5454

5555
private final MDBVal maxKey;
56+
private final MDBVal maxValue;
5657

5758
private final boolean matchValues;
58-
private GroupMatcher groupMatcher;
59+
private EntryMatcher matcher;
5960

6061
private final Txn txnRef;
6162

@@ -73,44 +74,56 @@ class LmdbRecordIterator implements RecordIterator {
7374

7475
private ByteBuffer minKeyBuf;
7576

76-
private ByteBuffer maxKeyBuf;
77+
private ByteBuffer minValueBuf;
7778

78-
private int lastResult;
79+
private final ByteBuffer maxKeyBuf;
80+
81+
private final ByteBuffer maxValueBuf;
7982

8083
private final long[] quad;
81-
private final long[] originalQuad;
84+
private final long[] patternQuad;
8285

8386
private boolean fetchNext = false;
8487

8588
private final StampedLongAdderLockManager txnLockManager;
8689

8790
private final Thread ownerThread = Thread.currentThread();
8891

89-
LmdbRecordIterator(TripleIndex index, boolean rangeSearch, long subj, long pred, long obj,
92+
private final int indexScore;
93+
94+
LmdbRecordIterator(TripleIndex index, int indexScore, long subj, long pred, long obj,
9095
long context, boolean explicit, Txn txnRef) throws IOException {
91-
this.subj = subj;
92-
this.pred = pred;
93-
this.obj = obj;
94-
this.context = context;
95-
this.originalQuad = new long[] { subj, pred, obj, context };
96+
this.patternQuad = new long[] { subj, pred, obj, context };
9697
this.quad = new long[] { subj, pred, obj, context };
9798
this.pool = Pool.get();
9899
this.keyData = pool.getVal();
99100
this.valueData = pool.getVal();
100101
this.index = index;
101-
if (rangeSearch) {
102+
this.indexScore = indexScore;
103+
// prepare min and max keys if index can be used
104+
// otherwise, leave as null to indicate full scan
105+
if (indexScore > 0) {
102106
minKeyBuf = pool.getKeyBuffer();
103-
index.getMinKey(minKeyBuf, subj, pred, obj, context);
107+
minValueBuf = pool.getKeyBuffer();
108+
index.getMinEntry(minKeyBuf, minValueBuf, subj, pred, obj, context);
104109
minKeyBuf.flip();
110+
minValueBuf.flip();
105111

106112
this.maxKey = pool.getVal();
113+
this.maxValue = pool.getVal();
107114
this.maxKeyBuf = pool.getKeyBuffer();
108-
index.getMaxKey(maxKeyBuf, subj, pred, obj, context);
115+
this.maxValueBuf = pool.getKeyBuffer();
116+
index.getMaxEntry(maxKeyBuf, maxValueBuf, subj, pred, obj, context);
109117
maxKeyBuf.flip();
118+
maxValueBuf.flip();
110119
this.maxKey.mv_data(maxKeyBuf);
120+
this.maxValue.mv_data(maxValueBuf);
111121
} else {
112122
minKeyBuf = null;
113123
this.maxKey = null;
124+
this.maxValue = null;
125+
this.maxKeyBuf = null;
126+
this.maxValueBuf = null;
114127
}
115128

116129
this.matchValues = subj > 0 || pred > 0 || obj > 0 || context >= 0;
@@ -147,6 +160,7 @@ public long[] next() {
147160
} catch (InterruptedException e) {
148161
throw new SailException(e);
149162
}
163+
int lastResult;
150164
try {
151165
if (closed) {
152166
log.debug("Calling next() on an LmdbRecordIterator that is already closed, returning null");
@@ -161,15 +175,21 @@ public long[] next() {
161175
// cursor must be positioned on last item, reuse minKeyBuf if available
162176
if (minKeyBuf == null) {
163177
minKeyBuf = pool.getKeyBuffer();
178+
minValueBuf = pool.getKeyBuffer();
164179
}
165180
minKeyBuf.clear();
166-
index.toKey(minKeyBuf, quad[0], quad[1], quad[2], quad[3]);
181+
index.toEntry(minKeyBuf, minValueBuf, quad[0], quad[1], quad[2], quad[3]);
167182
minKeyBuf.flip();
183+
minValueBuf.flip();
168184
keyData.mv_data(minKeyBuf);
169-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET);
170-
if (lastResult != MDB_SUCCESS) {
171-
// use MDB_SET_RANGE if key was deleted
172-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
185+
// use set range if entry was deleted
186+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
187+
if (lastResult == MDB_SUCCESS) {
188+
valueData.mv_data(minValueBuf);
189+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
190+
if (lastResult != MDB_SUCCESS) {
191+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_FIRST_DUP);
192+
}
173193
}
174194
if (lastResult != MDB_SUCCESS) {
175195
closeInternal(false);
@@ -180,14 +200,32 @@ public long[] next() {
180200
this.txnRefVersion = txnRef.version();
181201
}
182202

203+
boolean isDupValue = false;
183204
if (fetchNext) {
184-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
205+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_DUP);
206+
if (lastResult != MDB_SUCCESS) {
207+
// no more duplicates, move to next key
208+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_NODUP);
209+
} else {
210+
isDupValue = true;
211+
}
185212
fetchNext = false;
186213
} else {
187214
if (minKeyBuf != null) {
188215
// set cursor to min key
189216
keyData.mv_data(minKeyBuf);
190-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
217+
// set range on key is only required if less than the first two key elements are fixed
218+
lastResult = indexScore >= 2 ? MDB_SUCCESS
219+
: mdb_cursor_get(cursor, keyData, valueData, MDB_SET_RANGE);
220+
if (lastResult == MDB_SUCCESS) {
221+
valueData.mv_data(minValueBuf);
222+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_GET_BOTH_RANGE);
223+
if (lastResult != MDB_SUCCESS) {
224+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_FIRST_DUP);
225+
} else {
226+
isDupValue = indexScore >= 2;
227+
}
228+
}
191229
} else {
192230
// set cursor to first item
193231
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
@@ -196,14 +234,22 @@ public long[] next() {
196234

197235
while (lastResult == MDB_SUCCESS) {
198236
// if (maxKey != null && TripleStore.COMPARATOR.compare(keyData.mv_data(), maxKey.mv_data()) > 0) {
199-
if (maxKey != null && mdb_cmp(txn, dbi, keyData, maxKey) > 0) {
237+
int keyDiff;
238+
if (maxKey != null &&
239+
(keyDiff = isDupValue ? 0 : mdb_cmp(txn, dbi, keyData, maxKey)) >= 0
240+
&& (keyDiff > 0 || mdb_dcmp(txn, dbi, valueData, maxValue) > 0)) {
200241
lastResult = MDB_NOTFOUND;
201-
} else if (matches()) {
242+
} else if (notMatches(isDupValue)) {
202243
// value doesn't match search key/mask, fetch next value
203-
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT);
244+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_DUP);
245+
if (lastResult != MDB_SUCCESS) {
246+
// no more duplicates, move to next key
247+
lastResult = mdb_cursor_get(cursor, keyData, valueData, MDB_NEXT_NODUP);
248+
}
249+
isDupValue = false;
204250
} else {
205251
// Matching value found
206-
index.keyToQuad(keyData.mv_data(), originalQuad, quad);
252+
index.entryToQuad(keyData.mv_data(), valueData.mv_data(), patternQuad, quad);
207253
// fetch next value
208254
fetchNext = true;
209255
return quad;
@@ -216,13 +262,17 @@ public long[] next() {
216262
}
217263
}
218264

219-
private boolean matches() {
220-
221-
if (groupMatcher != null) {
222-
return !this.groupMatcher.matches(keyData.mv_data());
265+
private boolean notMatches(boolean testValueOnly) {
266+
if (matcher != null) {
267+
if (testValueOnly) {
268+
// already positioned on correct key, no need to match key again
269+
return !this.matcher.matchesValue(valueData.mv_data());
270+
}
271+
return !this.matcher.matches(keyData.mv_data(), valueData.mv_data());
223272
} else if (matchValues) {
224-
this.groupMatcher = index.createMatcher(subj, pred, obj, context);
225-
return !this.groupMatcher.matches(keyData.mv_data());
273+
// lazy init of group matcher
274+
this.matcher = index.createMatcher(patternQuad[0], patternQuad[1], patternQuad[2], patternQuad[3]);
275+
return !this.matcher.matches(keyData.mv_data(), valueData.mv_data());
226276
} else {
227277
return false;
228278
}
@@ -247,10 +297,13 @@ private void closeInternal(boolean maybeCalledAsync) {
247297
pool.free(valueData);
248298
if (minKeyBuf != null) {
249299
pool.free(minKeyBuf);
300+
pool.free(minValueBuf);
250301
}
251302
if (maxKey != null) {
252303
pool.free(maxKeyBuf);
253304
pool.free(maxKey);
305+
pool.free(maxValueBuf);
306+
pool.free(maxValue);
254307
}
255308
}
256309
} finally {

‎core/sail/lmdb/src/main/java/org/eclipse/rdf4j/sail/lmdb/Pool.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ class Pool {
2222
// thread-local pool instance
2323
private static final ThreadLocal<Pool> threadlocal = ThreadLocal.withInitial(Pool::new);
2424

25-
private final MDBVal[] valPool = new MDBVal[1024];
26-
private final ByteBuffer[] keyPool = new ByteBuffer[1024];
25+
private final MDBVal[] valPool = new MDBVal[2048];
26+
private final ByteBuffer[] keyPool = new ByteBuffer[2048];
2727
private final Statistics[] statisticsPool = new Statistics[512];
2828
private int valPoolIndex = -1;
2929
private int keyPoolIndex = -1;

0 commit comments

Comments
 (0)