Skip to content

Commit b754ff7

Browse files
jjezrajjezra
committed
WIP: TODO: add negative test
1 parent 00d9136 commit b754ff7

File tree

4 files changed

+196
-4
lines changed

4 files changed

+196
-4
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexScrubbing.java

-1
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,6 @@ private CompletableFuture<Boolean> indexScrubRangeOnly(@Nonnull FDBRecordStore s
142142
throw new UnsupportedOperationException("This index does not support scrubbing type " + scrubbingType);
143143
}
144144

145-
146145
return indexScrubRangeOnly(store, recordsScanned, index, tools, maintainer.isIdempotent());
147146
}
148147

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexMaintainer.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -761,7 +761,7 @@ public IndexScrubbingTools<?> getIndexScrubbingTools(final IndexScrubbingTools.S
761761
final Map<String, String> options = state.index.getOptions();
762762
if (Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_ENABLED)) ||
763763
Boolean.parseBoolean(options.get(LuceneIndexOptions.PRIMARY_KEY_SEGMENT_INDEX_V2_ENABLED))) {
764-
return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager);
764+
return new LuceneIndexScrubbingToolsMissing(partitioner, directoryManager, indexAnalyzerSelector);
765765
}
766766
return null;
767767
case DANGLING:

‎fdb-record-layer-lucene/src/main/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingToolsMissing.java

+14-2
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,14 @@ public class LuceneIndexScrubbingToolsMissing implements IndexScrubbingTools<FDB
6666
private final LucenePartitioner partitioner;
6767
@Nonnull
6868
private final FDBDirectoryManager directoryManager;
69+
@Nonnull
70+
private final LuceneAnalyzerCombinationProvider indexAnalyzerSelector;
6971

70-
public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager) {
72+
public LuceneIndexScrubbingToolsMissing(@Nonnull LucenePartitioner partitioner, @Nonnull FDBDirectoryManager directoryManager,
73+
@Nonnull LuceneAnalyzerCombinationProvider indexAnalyzerSelector) {
7174
this.partitioner = partitioner;
7275
this.directoryManager = directoryManager;
76+
this.indexAnalyzerSelector = indexAnalyzerSelector;
7377
}
7478

7579

@@ -183,7 +187,15 @@ private boolean isMissingIndexKey(FDBStoredRecord<Message> rec, Integer partitio
183187
// Here: iternal error, getIndexScrubbingTools should have indicated that scrub missing is not supported.
184188
throw new IllegalStateException("This scrubber should not have been used");
185189
}
186-
try (DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId)) {
190+
191+
try {
192+
// TODO: this is called to initilize the writer, else we get an exception at getDirectoryReader. Should it really be done for a RO operation?
193+
directoryManager.getIndexWriter(groupingKey, partitionId, indexAnalyzerSelector.provideIndexAnalyzer(""));
194+
} catch (IOException e) {
195+
throw LuceneExceptions.toRecordCoreException("failed getIndexWriter", e);
196+
}
197+
try {
198+
DirectoryReader directoryReader = directoryManager.getDirectoryReader(groupingKey, partitionId);
187199
final LucenePrimaryKeySegmentIndex.DocumentIndexEntry documentIndexEntry = segmentIndex.findDocument(directoryReader, rec.getPrimaryKey());
188200
if (documentIndexEntry == null) {
189201
// Here: the document had not been found in the PK segment index

‎fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScurbbingTest.java

+181
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
/*
2+
* LuceneIndexScurbbingTest.java
3+
*
4+
* This source file is part of the FoundationDB open source project
5+
*
6+
* Copyright 2015-2024 Apple Inc. and the FoundationDB project authors
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
21+
package com.apple.foundationdb.record.lucene;
22+
23+
import com.apple.foundationdb.record.lucene.directory.FDBDirectory;
24+
import com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository;
25+
import com.apple.foundationdb.record.lucene.directory.MockedLuceneIndexMaintainerFactory;
26+
import com.apple.foundationdb.record.lucene.directory.TestingIndexMaintainerRegistry;
27+
import com.apple.foundationdb.record.metadata.Index;
28+
import com.apple.foundationdb.record.metadata.expressions.GroupingKeyExpression;
29+
import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext;
30+
import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore;
31+
import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexScrubber;
32+
import com.apple.foundationdb.record.query.plan.QueryPlanner;
33+
import com.apple.foundationdb.record.util.pair.Pair;
34+
import com.apple.foundationdb.tuple.Tuple;
35+
import org.junit.jupiter.api.Test;
36+
import org.slf4j.Logger;
37+
import org.slf4j.LoggerFactory;
38+
39+
import java.io.IOException;
40+
import java.util.ArrayList;
41+
import java.util.Arrays;
42+
import java.util.List;
43+
import java.util.concurrent.TimeoutException;
44+
45+
import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX;
46+
import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createComplexDocument;
47+
import static com.apple.foundationdb.record.lucene.LuceneIndexTestUtils.createSimpleDocument;
48+
import static com.apple.foundationdb.record.lucene.directory.InjectedFailureRepository.Methods.LUCENE_READ_BLOCK;
49+
import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.COMPLEX_DOC;
50+
import static com.apple.foundationdb.record.provider.foundationdb.indexes.TextIndexTestUtils.SIMPLE_DOC;
51+
52+
class LuceneIndexScurbbingTest extends FDBLuceneTestBase {
53+
54+
private static final Logger LOGGER = LoggerFactory.getLogger(LuceneOnlineIndexingTest.class);
55+
56+
private void rebuildIndexMetaData(final FDBRecordContext context, final String document, final Index index) {
57+
Pair<FDBRecordStore, QueryPlanner> pair = LuceneIndexTestUtils.rebuildIndexMetaData(context, path, document, index, isUseCascadesPlanner());
58+
this.recordStore = pair.getLeft();
59+
this.planner = pair.getRight();
60+
}
61+
62+
private void disableIndex(Index index, String document) {
63+
try (final FDBRecordContext context = openContext()) {
64+
rebuildIndexMetaData(context, document, index);
65+
recordStore.markIndexDisabled(index).join();
66+
context.commit();
67+
}
68+
}
69+
70+
@Test
71+
void luceneIndexScrubMissingSimpleNoIssues() throws IOException {
72+
Index index = SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX;
73+
try (final FDBRecordContext context = openContext()) {
74+
rebuildIndexMetaData(context, SIMPLE_DOC, index);
75+
recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
76+
recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
77+
recordStore.saveRecord(createSimpleDocument(2222L, WAYLON + " who?", 1));
78+
context.commit();
79+
}
80+
try (final FDBRecordContext context = openContext()) {
81+
rebuildIndexMetaData(context, SIMPLE_DOC, index);
82+
recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
83+
recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
84+
recordStore.saveRecord(createSimpleDocument(2222L, WAYLON + " who?", 1));
85+
context.commit();
86+
}
87+
try (final FDBRecordContext context = openContext()) {
88+
rebuildIndexMetaData(context, SIMPLE_DOC, index);
89+
try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder()
90+
.setRecordStore(recordStore)
91+
.setIndex(index)
92+
.build()) {
93+
indexScrubber.scrubMissingIndexEntries();
94+
}
95+
}
96+
}
97+
98+
private String[] listFiles(Index index) throws IOException {
99+
try (FDBRecordContext context = openContext()) {
100+
if (index.getRootExpression() instanceof GroupingKeyExpression) {
101+
List<String> files = new ArrayList<>();
102+
for (int i = 0; i < 2; i++) {
103+
final FDBDirectory directory = new FDBDirectory(
104+
recordStore.indexSubspace(index).subspace(Tuple.from(i)),
105+
context, index.getOptions());
106+
files.addAll(Arrays.asList(directory.listAll()));
107+
}
108+
return files.toArray(new String[0]);
109+
} else {
110+
final FDBDirectory directory = new FDBDirectory(recordStore.indexSubspace(index), context, index.getOptions());
111+
return directory.listAll();
112+
}
113+
}
114+
}
115+
116+
@Test
117+
void luceneIndexScrubMissingSimple() throws IOException {
118+
Index index = SIMPLE_TEXT_SUFFIXES_WITH_PRIMARY_KEY_SEGMENT_INDEX;
119+
120+
long docId_1 = 1623L;
121+
long docId_2 = 1547L;
122+
long docId_3 = 2222L;
123+
long docId_4 = 899L;
124+
Tuple primaryKey_1 = Tuple.from(1, docId_1);
125+
Tuple primaryKey_2 = Tuple.from(1, docId_2);
126+
Tuple primaryKey_3 = Tuple.from(1, docId_3);
127+
Tuple primaryKey_4 = Tuple.from(1, docId_4);
128+
long startTime = System.currentTimeMillis();
129+
try (final FDBRecordContext context = openContext()) {
130+
rebuildIndexMetaData(context, COMPLEX_DOC, index);
131+
recordStore.saveRecord(createComplexDocument(docId_1, WAYLON, 1, startTime));
132+
recordStore.saveRecord(createComplexDocument(docId_2, WAYLON, 1, startTime + 1000));
133+
recordStore.saveRecord(createComplexDocument(docId_3, WAYLON + " who?", 1, startTime + 2000));
134+
recordStore.saveRecord(createComplexDocument(docId_4, ENGINEER_JOKE, 1, startTime + 3000));
135+
context.commit();
136+
}
137+
138+
final String[] files = listFiles(index);
139+
140+
try (final FDBRecordContext context = openContext()) {
141+
final FDBDirectory directory = new FDBDirectory(recordStore.indexSubspace(index), context, index.getOptions());
142+
directory.getPrimaryKeySegmentIndex().addOrDeletePrimaryKeyEntry(primaryKey_1.pack(), 0, 0, false, "_0");
143+
context.commit();
144+
}
145+
146+
147+
try (final FDBRecordContext context = openContext()) {
148+
rebuildIndexMetaData(context, SIMPLE_DOC, index);
149+
recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
150+
recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
151+
recordStore.saveRecord(createSimpleDocument(2222L, WAYLON + " who?", 1));
152+
context.commit();
153+
}
154+
155+
final InjectedFailureRepository injectedFailures = new InjectedFailureRepository();
156+
final TestingIndexMaintainerRegistry registry = new TestingIndexMaintainerRegistry();
157+
registry.overrideFactory(new MockedLuceneIndexMaintainerFactory(injectedFailures));
158+
159+
try (final FDBRecordContext context = openContext()) {
160+
Pair<FDBRecordStore, QueryPlanner> pair = LuceneIndexTestUtils.rebuildIndexMetaData(context, path, SIMPLE_DOC, index, isUseCascadesPlanner(), registry);
161+
this.recordStore = pair.getLeft();
162+
this.planner = pair.getRight();
163+
injectedFailures.addFailure(LUCENE_READ_BLOCK,
164+
new LuceneConcurrency.AsyncToSyncTimeoutException("Blah", new TimeoutException("Blah")), 3);
165+
166+
recordStore.saveRecord(createSimpleDocument(1623L, ENGINEER_JOKE, 2));
167+
recordStore.saveRecord(createSimpleDocument(1547L, WAYLON, 1));
168+
recordStore.saveRecord(createSimpleDocument(2222L, WAYLON + " who?", 1));
169+
context.commit();
170+
}
171+
try (final FDBRecordContext context = openContext()) {
172+
rebuildIndexMetaData(context, SIMPLE_DOC, index);
173+
try (OnlineIndexScrubber indexScrubber = OnlineIndexScrubber.newBuilder()
174+
.setRecordStore(recordStore)
175+
.setIndex(index)
176+
.build()) {
177+
indexScrubber.scrubMissingIndexEntries();
178+
}
179+
}
180+
}
181+
}

0 commit comments

Comments
 (0)