Update tests to prepare for concurrent execution

This readies tests for concurrent execution. It makes a few changes:

* The FDBDatabaseExtension is now scoped to the test instance instead of static. This allows us to create a new `FDBDatabaseFactory` and `FDBDatabase` for each test execution, which in turn means that we can execute tests that modify the factory's state concurrently, and we can have each database check for open contexts. This did mean that this needed to add a new method to create an `FDBDatabaseFactory` for testing (only) that accepted an already initialized FDB.
* One test introduced a write conflict range on the range `\x00` to `\xff`, so it would cause random other tests to fail if they were executed at the same time. I've narrowed the range of the conflict range so that it is now on a unique keyspace for that test.
* I introduced a few new test extensions that can be applied to every test. In particular, the `TestMdcExtension` updates the logger MDC with the test class, method name, and displey name so that test logs contain more information that can be used to correlate information. Meanwhile, the `ExceptionLoggingDetailsExtension` adds an exception handler for loggable exceptions that will log the logging details if a test fails with, say, a `RecordCoreException`, we have a way of understanding the underlying details.

Author: alecgrieser

.idea/compiler.xml

@@ -35,6 +35,8 @@ dependencies {
     testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:${junitVersion}"
     testRuntimeOnly "org.apache.logging.log4j:log4j-slf4j-impl:${log4jVersion}" // binding
     testImplementation "org.apache.logging.log4j:log4j-core:${log4jVersion}" // library
+    testCompileOnly "com.google.auto.service:auto-service:undefined"
+    testAnnotationProcessor "com.google.auto.service:auto-service:undefined"
 }
 
 publishing {

fdb-extensions/fdb-extensions.gradle

@@ -20,11 +20,11 @@
 
 package com.apple.foundationdb.async;
 
+import com.apple.foundationdb.test.TestExecutors;
 import org.junit.jupiter.api.Test;
 
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executor;
-import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.TimeUnit;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -35,8 +35,7 @@
  * Tests for {@link MoreAsyncUtil}.
  */
 public class MoreAsyncUtilTest {
-
-    static final Executor EXECUTOR = ForkJoinPool.commonPool();
+    static final Executor EXECUTOR = TestExecutors.defaultThreadPool();
 
     int count;
 

fdb-extensions/src/test/java/com/apple/foundationdb/async/MoreAsyncUtilTest.java

@@ -34,6 +34,8 @@
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 
 import javax.annotation.Nonnull;
 import java.util.ArrayList;
@@ -58,6 +60,7 @@
  * Tests for {@link RangeSet}.
  */
 @Tag(Tags.RequiresFDB)
+@Execution(ExecutionMode.CONCURRENT)
 public class RangeSetTest {
     @RegisterExtension
     static final TestDatabaseExtension dbExtension = new TestDatabaseExtension();
@@ -116,7 +119,7 @@ private void checkIncreasing() {
     }
 
     @BeforeEach
-    void setUp() throws Exception {
+    void setUp() {
         db = dbExtension.getDatabase();
         rsSubspace = rsSubspaceExtension.getSubspace();
         rs = new RangeSet(rsSubspace);

fdb-extensions/src/test/java/com/apple/foundationdb/async/RangeSetTest.java

@@ -27,6 +27,7 @@
 import com.apple.foundationdb.TransactionContext;
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.test.TestDatabaseExtension;
+import com.apple.foundationdb.test.TestExecutors;
 import com.apple.foundationdb.test.TestSubspaceExtension;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 import com.apple.foundationdb.tuple.Tuple;
@@ -35,12 +36,13 @@
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletionException;
-import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.ThreadLocalRandom;
 
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
@@ -54,6 +56,7 @@
  * Tests for {@link RankedSet}.
  */
 @Tag(Tags.RequiresFDB)
+@Execution(ExecutionMode.CONCURRENT)
 public class RankedSetTest {
     @RegisterExtension
     static final TestDatabaseExtension dbExtension = new TestDatabaseExtension();
@@ -288,7 +291,7 @@ public void rankAsThoughPresent() {
     //
 
     private RankedSet newRankedSet() {
-        RankedSet result = new RankedSet(rsSubspace, ForkJoinPool.commonPool(), config);
+        RankedSet result = new RankedSet(rsSubspace, TestExecutors.defaultThreadPool(), config);
         result.init(db).join();
         return result;
     }

fdb-extensions/src/test/java/com/apple/foundationdb/async/RankedSetTest.java

@@ -26,6 +26,7 @@
 import com.apple.foundationdb.async.AsyncUtil;
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.test.TestDatabaseExtension;
+import com.apple.foundationdb.test.TestExecutors;
 import com.apple.foundationdb.test.TestSubspaceExtension;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.Tags;
@@ -35,6 +36,8 @@
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -45,7 +48,6 @@
 import java.util.List;
 import java.util.Random;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.stream.Stream;
 
@@ -54,15 +56,14 @@
  */
 @Tag(Tags.RequiresFDB)
 @Tag(Tags.Slow)
+@Execution(ExecutionMode.CONCURRENT)
 public class RTreeModificationTest {
     private static final Logger logger = LoggerFactory.getLogger(RTreeModificationTest.class);
-    private static final boolean TRACE = false;
-
     private static final int NUM_TEST_RUNS = 5;
     private static final int NUM_SAMPLES = 10_000;
 
     @RegisterExtension
-    static final TestDatabaseExtension dbExtension = new TestDatabaseExtension(TRACE);
+    static final TestDatabaseExtension dbExtension = new TestDatabaseExtension();
     @RegisterExtension
     TestSubspaceExtension rtSubspace = new TestSubspaceExtension(dbExtension);
     @RegisterExtension
@@ -81,7 +82,7 @@ public void testAllDeleted(final RTree.Config config, final long seed, final int
         final RTreeScanTest.OnWriteCounters onWriteCounters = new RTreeScanTest.OnWriteCounters();
         final RTreeScanTest.OnReadCounters onReadCounters = new RTreeScanTest.OnReadCounters();
 
-        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), ForkJoinPool.commonPool(), config,
+        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), TestExecutors.defaultThreadPool(), config,
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, onWriteCounters,
                 onReadCounters);
         final long startTs = System.nanoTime();
@@ -139,7 +140,7 @@ public void testAllDeleted(final RTree.Config config, final long seed, final int
     @MethodSource("numSamplesAndNumDeletes")
     public void testRandomDeletes(@Nonnull final RTree.Config config, final long seed, final int numSamples, final int numDeletes) {
         final RTreeScanTest.OnReadCounters onReadCounters = new RTreeScanTest.OnReadCounters();
-        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), ForkJoinPool.commonPool(), config,
+        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), TestExecutors.defaultThreadPool(), config,
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, OnWriteListener.NOOP,
                 onReadCounters);
         final Item[] items = randomInserts(db, rTree, seed, numSamples);
@@ -201,7 +202,7 @@ public <T extends Node> CompletableFuture<T> onAsyncRead(@Nonnull final Completa
             }
         };
 
-        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), ForkJoinPool.commonPool(),
+        final RTree rTree = new RTree(rtSubspace.getSubspace(), rtSecondarySubspace.getSubspace(), TestExecutors.defaultThreadPool(),
                 new RTree.ConfigBuilder().build(),
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, OnWriteListener.NOOP,
                 onReadListener);

fdb-extensions/src/test/java/com/apple/foundationdb/async/rtree/RTreeModificationTest.java

@@ -27,6 +27,7 @@
 import com.apple.foundationdb.async.rtree.RTreeModificationTest.Item;
 import com.apple.foundationdb.subspace.Subspace;
 import com.apple.foundationdb.test.TestDatabaseExtension;
+import com.apple.foundationdb.test.TestExecutors;
 import com.apple.foundationdb.test.TestSubspaceExtension;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.Tags;
@@ -40,6 +41,8 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
@@ -54,7 +57,6 @@
 import java.util.Queue;
 import java.util.Random;
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ForkJoinPool;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Predicate;
 import java.util.stream.Stream;
@@ -64,14 +66,14 @@
  */
 @Tag(Tags.RequiresFDB)
 @Tag(Tags.Slow)
+@Execution(ExecutionMode.CONCURRENT)
 public class RTreeScanTest  {
     private static final Logger logger = LoggerFactory.getLogger(RTreeScanTest.class);
-    private static final boolean TRACE = false;
 
     private static final int NUM_SAMPLES = 10_000;
     private static final int NUM_QUERIES = 100;
     @RegisterExtension
-    static TestDatabaseExtension dbExtension = new TestDatabaseExtension(TRACE);
+    static TestDatabaseExtension dbExtension = new TestDatabaseExtension();
 
     private static Database db;
     private static Subspace rtSubspace;
@@ -84,7 +86,7 @@ public static void setUpDb() {
         db = dbExtension.getDatabase();
         rtSubspace = new TestSubspaceExtension(dbExtension).getSubspace();
         rtSecondarySubspace = new TestSubspaceExtension(dbExtension).getSubspace();
-        final RTree rTree = new RTree(rtSubspace, rtSecondarySubspace, ForkJoinPool.commonPool(), RTree.DEFAULT_CONFIG,
+        final RTree rTree = new RTree(rtSubspace, rtSecondarySubspace, TestExecutors.defaultThreadPool(), RTree.DEFAULT_CONFIG,
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, OnWriteListener.NOOP,
                 OnReadListener.NOOP);
         final Item[] items1 = RTreeModificationTest.randomInsertsWithNulls(db, rTree, 0L, NUM_SAMPLES / 2);
@@ -158,7 +160,7 @@ public void queryWithFilters(@Nonnull final RTree.Rectangle query) {
 
         final OnReadCounters onReadCounters = new OnReadCounters();
 
-        final RTree rt = new RTree(rtSubspace, rtSecondarySubspace, ForkJoinPool.commonPool(), RTree.DEFAULT_CONFIG,
+        final RTree rt = new RTree(rtSubspace, rtSecondarySubspace, TestExecutors.defaultThreadPool(), RTree.DEFAULT_CONFIG,
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, OnWriteListener.NOOP,
                 onReadCounters);
 
@@ -206,7 +208,7 @@ public void queryTopNWithFilters(@Nonnull final RTree.Rectangle query) {
             }
         }
         final OnReadCounters onReadCounters = new OnReadCounters();
-        final RTree rt = new RTree(rtSubspace, rtSecondarySubspace, ForkJoinPool.commonPool(), RTree.DEFAULT_CONFIG,
+        final RTree rt = new RTree(rtSubspace, rtSecondarySubspace, TestExecutors.defaultThreadPool(), RTree.DEFAULT_CONFIG,
                 RTreeHilbertCurveHelpers::hilbertValue, NodeHelpers::newSequentialNodeId, OnWriteListener.NOOP,
                 onReadCounters);
         final AtomicLong nresults = new AtomicLong(0L);

fdb-extensions/src/test/java/com/apple/foundationdb/async/rtree/RTreeScanTest.java

@@ -28,6 +28,8 @@
 import org.junit.jupiter.api.extension.ExtensionContext;
 
 import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.util.Objects;
 
 /**
  * Test extension to use to get a connection to an FDB {@link Database}. This handles setting up the FDB
@@ -48,16 +50,14 @@ public class TestDatabaseExtension implements BeforeAllCallback, AfterAllCallbac
     private static final int MIN_API_VERSION = 630;
     private static final int MAX_API_VERSION = 710;
     private static final String API_VERSION_PROPERTY = "com.apple.foundationdb.apiVersion";
+    private static final boolean TRACE = false;
+
+    @Nullable
+    private static volatile FDB fdb;
 
-    private final boolean trace;
     private Database db;
 
     public TestDatabaseExtension() {
-        this(false);
-    }
-
-    public TestDatabaseExtension(boolean trace) {
-        this.trace = trace;
     }
 
     public static int getAPIVersion() {
@@ -69,17 +69,28 @@ public static int getAPIVersion() {
         return apiVersion;
     }
 
-    @Override
-    public void beforeAll(final ExtensionContext extensionContext) {
-        if (!FDB.isAPIVersionSelected()) {
-            FDB fdb = FDB.selectAPIVersion(getAPIVersion());
-            if (trace) {
-                NetworkOptions options = fdb.options();
-                options.setTraceEnable("/tmp");
-                options.setTraceLogGroup("fdb_extensions_tests");
+    @Nonnull
+    private static FDB getFDB() {
+        if (fdb == null) {
+            synchronized (TestDatabaseExtension.class) {
+                if (fdb == null) {
+                    FDB inst = FDB.selectAPIVersion(getAPIVersion());
+                    if (TRACE) {
+                        NetworkOptions options = inst.options();
+                        options.setTraceEnable("/tmp");
+                        options.setTraceLogGroup("fdb_extensions_tests");
+                    }
+                    inst.setUnclosedWarning(true);
+                    fdb = inst;
+                }
             }
-            fdb.setUnclosedWarning(true);
         }
+        return Objects.requireNonNull(fdb);
+    }
+
+    @Override
+    public void beforeAll(final ExtensionContext extensionContext) {
+        getFDB();
     }
 
     @Nonnull

fdb-extensions/src/test/java/com/apple/foundationdb/test/TestDatabaseExtension.java

@@ -0,0 +1,65 @@
+/*
+ * TestExecutors.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.test;
+
+import javax.annotation.Nonnull;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Executors to use during testing.
+ */
+public class TestExecutors {
+    /**
+     * Thread factory for creating threads used by test thread pools.
+     */
+    public static class TestThreadFactory implements ThreadFactory {
+        private final String namePrefix;
+        private final AtomicInteger count;
+
+        public TestThreadFactory(@Nonnull String namePrefix) {
+            this.namePrefix = namePrefix;
+            this.count = new AtomicInteger();
+        }
+
+        @Override
+        public Thread newThread(final Runnable r) {
+            Thread t = new Thread(r);
+            t.setDaemon(true);
+            t.setName(namePrefix + "-" + count.incrementAndGet());
+            return t;
+        }
+    }
+
+    @Nonnull
+    private static final Executor DEFAULT_THREAD_POOL = newThreadPool("fdb-unit-test");
+
+    public static Executor newThreadPool(@Nonnull String namePrefix) {
+        return Executors.newCachedThreadPool(new TestThreadFactory(namePrefix));
+    }
+
+    @Nonnull
+    public static Executor defaultThreadPool() {
+        return DEFAULT_THREAD_POOL;
+    }
+}