Add support for testing against multiple FDB clusters

.gitignore

@@ -80,6 +80,7 @@ protogen/
 
 # Local FDB settings
 fdb-environment.properties
+fdb-environment.yaml
 
 # Docker local support
 run/

actions/setup-fdb/action.yml

@@ -31,12 +31,57 @@ runs:
     - name: Install FDB Server
       shell: bash
       run: sudo dpkg -i ~/.fdb-cache/${{ steps.fdb_filenames.outputs.client_deb }} ~/.fdb-cache/${{ steps.fdb_filenames.outputs.server_deb }}
-    - name: Fix FDB Network Addresses
+    - name: Stop default fdb
       shell: bash
-      run: sudo sed -i -e "s/public_address = auto:\$ID/public_address = 127.0.0.1:\$ID/g" -e "s/listen_address = public/listen_address = 0.0.0.0:\$ID/g" /etc/foundationdb/foundationdb.conf
-    - name: Start FDB Server
+      run: sudo service foundationdb stop
+
+    - name: Create cluster1 config
+      shell: bash
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb1.conf
+
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb1.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb1/" \
+        -e "s/fdbserver.4500/fdbserver.4600/g" \
+        /etc/foundationdb/foundationdb1.conf
+
+        sudo bash -c "echo 'fdb1:$(mktemp -u XXXXXXXX)@127.0.0.1:4600' > /etc/foundationdb/fdb1.cluster"
+    - name: Create cluster2 config
       shell: bash
-      run: sudo /usr/lib/foundationdb/fdbmonitor /etc/foundationdb/foundationdb.conf --daemonize
-    - name: Switch FDB to SSD
+      run: |
+        sudo cp /etc/foundationdb/foundationdb.conf /etc/foundationdb/foundationdb2.conf
+
+        sudo sed -i -e "s/\/etc\/foundationdb\/fdb.cluster/\/etc\/foundationdb\/fdb2.cluster/g" \
+        -e "s/\/var\/log\/foundationdb/\/var\/log\/foundationdb2/" \
+        -e "s/fdbserver.4500/fdbserver.4700/g" \
+        /etc/foundationdb/foundationdb2.conf
+
+        sudo bash -c "echo 'fdb2:$(mktemp -u XXXXXXXX)@127.0.0.1:4700' > /etc/foundationdb/fdb2.cluster"
+
+    - name: create dirs & set permissions
       shell: bash
-      run: fdbcli --exec "configure single ssd storage_migration_type=aggressive; status"
+      run: |
+        sudo mkdir /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chown foundationdb:foundationdb /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster /var/log/foundationdb1 /var/log/foundationdb2
+        sudo chmod 664 /etc/foundationdb/fdb1.cluster /etc/foundationdb/fdb2.cluster
+        sudo chmod 700 /var/log/foundationdb1 /var/log/foundationdb2
+
+    - name: Start FDB Server 1
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb1.conf --lockfile /var/run/fdbmonitor1.pid --loggroup fdb1 --daemonize
+    - name: Start FDB Server 2
+      shell: bash
+      run: sudo /usr/lib/foundationdb/fdbmonitor --conffile /etc/foundationdb/foundationdb2.conf --lockfile /var/run/fdbmonitor2.pid --loggroup fdb2 --daemonize
+
+    - name: Switch FDB 1 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb1.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Switch FDB 2 to SSD
+      shell: bash
+      run: fdbcli -C /etc/foundationdb/fdb2.cluster --exec "configure new single ssd storage_migration_type=aggressive; status"
+    - name: Create fdb-environment.yaml
+      shell: bash
+      run: |
+        echo "clusterFiles: " >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb1.cluster" >> fdb-environment.yaml
+        echo "  - /etc/foundationdb/fdb2.cluster" >> fdb-environment.yaml

build.gradle

@@ -18,6 +18,10 @@
  * limitations under the License.
  */
 
+import org.yaml.snakeyaml.Yaml
+
+import org.apache.tools.ant.taskdefs.condition.Os
+
 buildscript {
     repositories {
         if (Boolean.parseBoolean(mavenLocalEnabled)) {
@@ -29,6 +33,7 @@ buildscript {
 
     dependencies {
         classpath 'org.jboss.tattletale:tattletale:1.2.0.Beta2'
+        classpath libs.snakeyaml
     }
 }
 
@@ -341,6 +346,8 @@ if (!JavaVersion.current().isJava8Compatible()) {
     throw new Exception("Java 8 is required to build fdb-record-layer")
 }
 
+// fdb-environment.properties is the old way we configured the library path and cluster file, it does not scale well
+// to multiple cluster files, so is being replaced with a yaml file.
 def fdbEnvironmentFile = new File("${rootProject.projectDir}/fdb-environment.properties")
 if (fdbEnvironmentFile.exists()) {
     fdbEnvironmentFile.eachLine { line ->
@@ -356,3 +363,15 @@ if (!ext.fdbEnvironment.isEmpty()) {
         }
     }
 }
+
+def fdbEnvironmentYamlFile = new File("${rootProject.projectDir}/fdb-environment.yaml")
+if (fdbEnvironmentYamlFile.exists()) {
+    def libraryPath = new Yaml().loadAll(fdbEnvironmentYamlFile.newInputStream()).first().libraryPath
+    def libraryPathVariable = Os.isFamily(Os.FAMILY_MAC) ? "DYLD_LIBRARY_PATH" : "LD_LIBRARY_PATH"
+    allprojects {
+        tasks.withType(Test) { task ->
+            task.environment([(libraryPathVariable): libraryPath])
+            task.environment(FDB_ENVIRONMENT_YAML: fdbEnvironmentYamlFile.absolutePath)
+        }
+    }
+}

docker-local/start.sh

@@ -95,6 +95,13 @@ FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE}
 ${LIBRARY_PATH}=${RUNDIR}
 EOF
 
+cat >${ROOTDIR}/fdb-environment.yaml <<EOF
+# docker-local
+clusterFiles:
+  - ${FDB_CLUSTER_FILE}
+libraryPath: ${RUNDIR}
+EOF
+
 cat ${ROOTDIR}/fdb-environment.properties
 
 echo "Docker-based FDB cluster is now up."

docs/sphinx/source/Building.md

@@ -63,7 +63,24 @@ If you enable the local repo in whatever uses the Record Layer, the following wi
 ./gradlew publishToMavenLocal -PpublishBuild=true
 ```
 
-## Configuring tests for a non-standard FDB cluster file location
+## Configuring tests for a non-standard FDB cluster file location, or to run against multiple clusters
+
+If a file `fdb-environment.yaml` exists in the root of the working directory, it contains the configuration for the C API library
+and a list of cluster files. Most tests will chose randomly from the provided cluster files when testing.
+
+Here is an example file:
+``` yaml
+libraryPath: /Users/scott/fdb/bin/fdb-server-7.3.42-macos_arm64/lib
+clusterFiles:
+  - /Users/scott/fdb/data/fdb-one.cluster
+  - /Users/scott/fdb/data/fdb-two.cluster
+```
+
+
+
+### deprecated properties file
+
+[ this is being replaced by the yaml file described above, to better support multiple cluster files ]
 
 If a file `fdb-environment.properties` exists in the root of the working directory, it contains environment variables that specify where to find
 the local FDB. These settings will apply when running inside IntelliJ as well as from a command line and so are easier to manage than a shell script.

fdb-extensions/fdb-extensions.gradle

@@ -32,11 +32,13 @@ dependencies {
 
     testImplementation project(':fdb-test-utils')
     testImplementation(libs.bundles.test.impl)
+    testImplementation(libs.snakeyaml)
     testRuntimeOnly(libs.bundles.test.runtime)
     testCompileOnly(libs.bundles.test.compileOnly)
     testAnnotationProcessor(libs.autoService)
 
     testFixturesImplementation(libs.bundles.test.impl)
+    testFixturesImplementation(libs.snakeyaml)
     testFixturesCompileOnly(libs.bundles.test.compileOnly)
     testFixturesImplementation(libs.slf4j.api)
     testFixturesAnnotationProcessor(libs.autoService)

fdb-record-layer-core/fdb-record-layer-core.gradle

@@ -44,6 +44,7 @@ dependencies {
     testImplementation(libs.junit.platform)
     testImplementation(libs.protobuf.util)
     testCompileOnly(libs.spotbugs.annotations)
+    testImplementation(libs.snakeyaml)
 
     testAnnotationProcessor(libs.autoService)
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/BlockingInAsyncDetectionTest.java

@@ -24,6 +24,7 @@
 import com.apple.foundationdb.record.RecordCoreException;
 import com.apple.foundationdb.record.TestHelpers;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.test.Tags;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
@@ -57,7 +58,7 @@ void testBlockingInAsyncException() {
         // Make sure that we aren't holding on to previously created databases
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         assertEquals(BlockingInAsyncDetection.IGNORE_COMPLETE_EXCEPTION_BLOCKING, database.getBlockingInAsyncDetection());
         assertThrows(BlockingInAsyncException.class, () -> callAsyncBlocking(database));
     }
@@ -68,7 +69,7 @@ void testBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.IGNORE_COMPLETE_WARN_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> {
                     callAsyncBlocking(database, true);
@@ -82,7 +83,7 @@ void testCompletedBlockingInAsyncWarning() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_IN_ASYNC_CONTEXT_MESSAGE,
                 () -> database.asyncToSync(new FDBStoreTimer(), FDBStoreTimer.Waits.WAIT_ERROR_CHECK,
                         CompletableFuture.supplyAsync(() ->
@@ -95,7 +96,7 @@ void testBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, MoreAsyncUtil.delayedFuture(200L, TimeUnit.MILLISECONDS, database.getScheduledExecutor())));
     }
@@ -106,7 +107,7 @@ void testCompletedBlockingCreatingAsyncDetection() {
         factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.WARN_COMPLETE_EXCEPTION_BLOCKING);
         factory.clear();
 
-        FDBDatabase database = factory.getDatabase();
+        FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         TestHelpers.assertDidNotLog(FDBDatabase.class, FDBDatabase.BLOCKING_RETURNING_ASYNC_MESSAGE,
                 () -> returnAnAsync(database, CompletableFuture.completedFuture(10L)));
     }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBDatabaseTest.java

@@ -36,6 +36,7 @@
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.BooleanSource;
 import com.apple.test.Tags;
@@ -54,6 +55,7 @@
 
 import javax.annotation.Nonnull;
 import java.io.IOException;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
@@ -66,6 +68,7 @@
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
 import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -240,7 +243,7 @@ void testJoinNowOnNonCompletedFuture(BlockingInAsyncDetection behavior) {
         if (behavior.throwExceptionOnBlocking()) {
             assertThrows(BlockingInAsyncException.class, () -> database.joinNow(new CompletableFuture<>()));
         } else {
-            FDBDatabase database2 = factory.getDatabase();
+            FDBDatabase database2 = factory.getDatabase(database.getClusterFile());
             TestHelpers.assertLogs(FDBDatabase.class, FDBDatabase.BLOCKING_FOR_FUTURE_MESSAGE, () -> {
                 long val = database2.joinNow(MoreAsyncUtil.delayedFuture(100, TimeUnit.MILLISECONDS, database2.getScheduledExecutor())
                         .thenApply(vignore -> 1066L));
@@ -494,4 +497,31 @@ void cannotChangeAPIVersionAfterInit() {
             assertEquals(initApiVersion, database.getAPIVersion());
         }
     }
+
+    @Test
+    void canAccessMultipleClusters() {
+        FDBTestEnvironment.assumeClusterCount(2);
+        final FDBDatabase database0 = dbExtension.getDatabase(0);
+        final FDBDatabase database1 = dbExtension.getDatabase(1);
+        final byte[] key = Tuple.from(UUID.randomUUID()).pack();
+        final byte[] value0 = Tuple.from("cluster0").pack();
+        final byte[] value1 = Tuple.from("cluster1").pack();
+        try (FDBRecordContext context0 = database0.openContext()) {
+            context0.ensureActive().set(key, value0);
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertNull(context1.ensureActive().get(key).join());
+            context1.ensureActive().set(key, value1);
+            context1.commit();
+        }
+        try (FDBRecordContext context0 = database0.openContext()) {
+            assertArrayEquals(value0, context0.ensureActive().get(key).join());
+            context0.commit();
+        }
+        try (FDBRecordContext context1 = database1.openContext()) {
+            assertArrayEquals(value1, context1.ensureActive().get(key).join());
+            context1.commit();
+        }
+    }
 }

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordContextTest.java

@@ -34,6 +34,7 @@
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
 import com.apple.foundationdb.record.util.pair.Pair;
 import com.apple.foundationdb.subspace.Subspace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.ByteArrayUtil;
 import com.apple.foundationdb.tuple.ByteArrayUtil2;
 import com.apple.foundationdb.tuple.Tuple;
@@ -433,7 +434,7 @@ public void logWithoutSettingId() {
             FDBDatabaseFactory factory = fdb.getFactory();
             factory.setTransactionIsTracedSupplier(() -> false);
             factory.clear();
-            fdb = factory.getDatabase();
+            fdb = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
         }
         try (FDBRecordContext context = fdb.openContext()) {
             RecordCoreException err = assertThrows(RecordCoreException.class, context::logTransaction);
@@ -665,7 +666,8 @@ public void contextExecutor() {
             factory.setContextExecutor(exec -> new ThreadIdRestoringExecutor(exec, myThreadId));
             factory.clear();
 
-            FDBDatabase database = factory.getDatabase();
+
+            FDBDatabase database = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
             try (FDBRecordContext context = database.openContext()) {
                 context.ensureActive().get(new byte[] { 0 }).thenAccept( value -> {
                     assertEquals(myThreadId, ThreadId.get());

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStorePerformanceTest.java

@@ -38,6 +38,7 @@
 import com.apple.foundationdb.record.provider.foundationdb.keyspace.KeySpacePath;
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
 import com.apple.foundationdb.record.test.TestKeySpace;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.foundationdb.util.StringUtils;
 import com.apple.test.Tags;
@@ -164,7 +165,7 @@ public void setup() {
     public void createMetaData() {
         FDBDatabaseFactory factory = dbExtension.getDatabaseFactory();
         factory.setDirectoryCacheSize(databaseParameters.pathCache);
-        fdb = factory.getDatabase();
+        fdb = factory.getDatabase(FDBTestEnvironment.randomClusterFile());
 
         RecordMetaDataBuilder metaDataBuilder = RecordMetaData.newBuilder().setRecords(TestRecords1Proto.getDescriptor());
         metaDataBuilder.setSplitLongRecords(databaseParameters.splitRecords);

fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBReverseDirectoryCacheTest.java

@@ -34,6 +34,7 @@
 import com.apple.foundationdb.record.test.FDBDatabaseExtension;
 import com.apple.foundationdb.record.test.TestKeySpace;
 import com.apple.foundationdb.record.test.TestKeySpacePathManagerExtension;
+import com.apple.foundationdb.test.FDBTestEnvironment;
 import com.apple.foundationdb.record.util.pair.Pair;
 import com.apple.foundationdb.tuple.Tuple;
 import com.apple.test.Tags;
@@ -387,8 +388,9 @@ public void testMapPathKeysConflict() throws Exception {
     public void testMapPathKeysConflictMultipleDatabaseObjects() throws Exception {
         // this is to simulate multiple VMs
         final int parallelism = 20;
+        final String clusterFile = FDBTestEnvironment.randomClusterFile();
         testParallelReverseDirectoryCache(parallelism, true,
-                () -> new FDBDatabase(dbExtension.getDatabaseFactory(), null));
+                () -> new FDBDatabase(dbExtension.getDatabaseFactory(), clusterFile));
     }
 
     @Test
@@ -404,8 +406,9 @@ public void testCacheInitConflict() throws Exception {
     public void testCacheInitConflictMultipleDatabaseObjects() throws Exception {
         // this is to simulate multiple VMs
         final int parallelism = 20;
+        final String clusterFile = FDBTestEnvironment.randomClusterFile();
         testParallelReverseDirectoryCache(parallelism, false,
-                () -> new FDBDatabase(dbExtension.getDatabaseFactory(), null));
+                () -> new FDBDatabase(dbExtension.getDatabaseFactory(), clusterFile));
     }
 
     private void testParallelReverseDirectoryCache(int parallelism, boolean preInitReverseDirectoryCache,
@@ -458,8 +461,8 @@ private void runParallelCodeOnEmptyDB(int parallelism,
         final Supplier<BlockingInAsyncDetection> blockingInAsyncDetectionSupplier = factory.getBlockingInAsyncDetectionSupplier();
         try {
             factory.setBlockingInAsyncDetection(BlockingInAsyncDetection.DISABLED);
-            // Get a fresh new one
-            fdb = factory.getDatabase();
+            // Get a fresh new one (but the same cluster as was wiped)
+            fdb = factory.getDatabase(fdb.getClusterFile());
 
             final Executor executor = new ForkJoinPool(parallelism + 1);
             final Semaphore lock = new Semaphore(parallelism);
@@ -590,7 +593,7 @@ public void testUniqueCachePerDatabase() throws Exception {
         factory.clear();
 
         // Get a fresh new one
-        fdb = factory.getDatabase();
+        fdb = factory.getDatabase(fdb.getClusterFile());
         cache = fdb.getReverseDirectoryCache();
 
         // In the hopes to ensure that re-creating the entries that we previously created