wip

Author: jjezra

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexGetMetadataInfoTest.java

@@ -24,7 +24,6 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase;
 import com.apple.foundationdb.record.provider.foundationdb.IndexOperationResult;
-import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer;
 import com.apple.foundationdb.tuple.Tuple;
 import com.google.protobuf.ByteString;
 import org.hamcrest.Matchers;
@@ -38,7 +37,6 @@
 import java.time.Instant;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -105,7 +103,9 @@ void getMetadataPartitioned(boolean justPartitionInfo, boolean isGrouped) {
                 dataModel.saveRecords(10, start, context, i / 3);
                 commit(context);
             }
-            explicitMergeIndex(dataModel);
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.explicitMergeIndex(context, timer);
+            }
         }
 
         final Set<Tuple> groupingKeys = isGrouped ? dataModel.groupingKeys() : Set.of(Tuple.from());
@@ -150,7 +150,9 @@ void getMetadataAfterDelete() {
                 dataModel.saveRecords(10, start, context, i / 3);
                 commit(context);
             }
-            explicitMergeIndex(dataModel);
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.explicitMergeIndex(context, timer);
+            }
         }
 
         final Tuple groupingKey = Tuple.from();
@@ -229,17 +231,4 @@ private static void assertLessThan(final ByteString lesserOne, final ByteString
     private static int segmentCountToFileCount(final int segmentCount) {
         return segmentCount * 4 + 1;
     }
-
-    private void explicitMergeIndex(LuceneIndexTestDataModel dataModel) {
-        try (FDBRecordContext context = openContext()) {
-            FDBRecordStore recordStore = Objects.requireNonNull(dataModel.schemaSetup.apply(context));
-            try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder()
-                    .setRecordStore(recordStore)
-                    .setIndex(dataModel.index)
-                    .setTimer(timer)
-                    .build()) {
-                indexBuilder.mergeIndex();
-            }
-        }
-    }
 }

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexScrubbingTest.java

@@ -91,12 +91,15 @@ void luceneIndexScrubMissingDataModelNoIssues(boolean isSynthetic, boolean isGro
                 .setPartitionHighWatermark(isPartitioned ? 10 : 0)
                 .build();
 
-        try (final FDBRecordContext context = openContext()) {
-            FDBRecordStore store = dataModel.createOrOpenRecordStore(context);
-            dataModel.saveRecords(15, 1007, context, 1);
-            context.commit();
+        for (int i = 0; i < 14; i++) {
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.saveRecords(3, i * 1007, context, i / 6);
+                context.commit();
+            }
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.explicitMergeIndex(context, timer);
+            }
         }
-
         try (final FDBRecordContext context = openContext()) {
             FDBRecordStore store = dataModel.createOrOpenRecordStore(context);
             boolean atLeastOnce = false;
@@ -170,15 +173,25 @@ void luceneIndexScrubMissingDataModel(boolean isSynthetic, boolean isGrouped, bo
         final InjectedFailureRepository injectedFailures = new InjectedFailureRepository();
         registry.overrideFactory(new MockedLuceneIndexMaintainerFactory(injectedFailures));
 
+        for (int i = 0; i < 14; i++) {
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.saveRecords(3, i * 1007, context, i / 6);
+                context.commit();
+            }
+            try (final FDBRecordContext context = openContext()) {
+                dataModel.explicitMergeIndex(context, timer);
+            }
+        }
+
         try (final FDBRecordContext context = openContext()) {
             // Write some documents
-            dataModel.saveRecords(15, 1007, context, 1);
+            dataModel.saveRecords(15, 10, context, 1);
             // Trigger failures
             injectedFailures.setFlag(LUCENE_MAINTAINER_SKIP_INDEX_UPDATE);
             // Some overwrites (with index update failures)
-            dataModel.saveRecords(5, 1007, context, 1);
+            dataModel.saveRecords(5, 10, context, 1);
             // some new (with index update failures)
-            dataModel.saveRecords(5, 7007, context, 1);
+            dataModel.saveRecords(5, 50, context, 1);
             injectedFailures.setFlag(LUCENE_MAINTAINER_SKIP_INDEX_UPDATE, false);
             context.commit();
         }

fdb-record-layer-lucene/src/test/java/com/apple/foundationdb/record/lucene/LuceneIndexTestDataModel.java

@@ -32,6 +32,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore;
 import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer;
 import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord;
+import com.apple.foundationdb.record.provider.foundationdb.OnlineIndexer;
 import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
@@ -331,6 +332,18 @@ static RecordMetaDataBuilder createBaseMetaDataBuilder() {
         return metaDataBuilder;
     }
 
+    public void explicitMergeIndex(final FDBRecordContext context, @Nullable FDBStoreTimer timer) {
+        FDBRecordStore recordStore = Objects.requireNonNull(schemaSetup.apply(context));
+        try (OnlineIndexer indexBuilder = OnlineIndexer.newBuilder()
+                .setRecordStore(recordStore)
+                .setIndex(index)
+                .setTimer(timer)
+                .build()) {
+            indexBuilder.mergeIndex();
+        }
+    }
+
+
     public Integer nextInt(final int bound) {
         return random.nextInt(bound);
     }