From 9ab47e50bf4c24035dbd64dd0769336a28800756 Mon Sep 17 00:00:00 2001 From: Sushant Mane Date: Mon, 3 Feb 2025 13:54:16 -0800 Subject: [PATCH] [test] Update Venice cluster initialization to replace deprecated APIs (#1495) Refactored cluster initialization in tests by replacing calls to deprecated APIs with overloads that had numerous method parameters, using `VeniceClusterCreateOptions` and `VeniceMultiRegionClusterCreateOptions` instead. This change simplifies test setup, improves readability, and reduces complexity by using builder-based initialization. Also added support for passing `DynamicAccessController` to Venice controllers in E2E test setups to enable testing of access control code paths. --- .../venice/VeniceClusterInitializer.java | 12 +- .../consumer/ConsumerIntegrationTest.java | 5 +- .../TestBootstrappingChangelogConsumer.java | 25 +-- .../consumer/TestChangelogConsumer.java | 25 +-- .../venice/controller/AdminToolE2ETest.java | 26 +-- .../controller/TestAdminToolEndToEnd.java | 12 +- .../venice/controller/TestFabricBuildout.java | 26 +-- .../venice/controller/TestHAASController.java | 50 ++++- .../TestHolisticSeverHealthCheck.java | 11 +- ...toreRepartitioningWithMultiDataCenter.java | 26 +-- ...stParentControllerWithMultiDataCenter.java | 28 +-- .../controller/TestStartMultiControllers.java | 7 +- .../TestTopicRequestOnHybridDelete.java | 5 +- .../VeniceParentHelixAdminTest.java | 55 +++--- .../TestAdminSparkServerWithMultiServers.java | 7 +- .../ActiveActiveReplicationForHybridTest.java | 27 +-- .../BlobP2PTransferAmongServersTest.java | 8 +- .../venice/endToEnd/CheckSumTest.java | 12 +- .../DaVinciClientRecordTransformerTest.java | 12 +- .../venice/endToEnd/DaVinciClientTest.java | 12 +- .../endToEnd/DaVinciClusterAgnosticTest.java | 26 +-- .../venice/endToEnd/DaVinciComputeTest.java | 12 +- .../DaVinciLiveUpdateSuppressionTest.java | 12 +- .../venice/endToEnd/DataRecoveryTest.java | 27 +-- .../endToEnd/NearlineE2ELatencyTest.java | 25 +-- .../endToEnd/OneTouchDataRecoveryTest.java | 25 +-- .../PartialUpdateClusterConfigTest.java | 28 +-- .../venice/endToEnd/PartialUpdateTest.java | 28 +-- .../venice/endToEnd/PushJobDetailsTest.java | 27 +-- .../PushStatusStoreMultiColoTest.java | 27 +-- .../endToEnd/RocksDBPlainTableTest.java | 5 +- .../endToEnd/StoreMetadataRecoveryTest.java | 25 +-- .../endToEnd/TestActiveActiveIngestion.java | 27 +-- ...TestActiveActiveReplicationForIncPush.java | 27 +-- ...ReplicationWithDelayedLeaderPromotion.java | 25 +-- ...ActiveActiveReplicationWithDownRegion.java | 27 +-- ...TestBackupVersionDatabaseOptimization.java | 12 +- .../venice/endToEnd/TestBatchForRocksDB.java | 5 +- .../TestBatchReportIncrementalPush.java | 28 +-- .../endToEnd/TestDeferredVersionSwap.java | 27 +-- .../endToEnd/TestDumpIngestionContext.java | 28 +-- ...tFatalDataValidationExceptionHandling.java | 12 +- .../endToEnd/TestHelixCustomizedView.java | 11 +- .../linkedin/venice/endToEnd/TestHybrid.java | 49 ++++- .../endToEnd/TestHybridMultiRegion.java | 26 +-- .../venice/endToEnd/TestHybridQuota.java | 12 +- .../endToEnd/TestLeaderReplicaFailover.java | 14 +- .../TestMaterializedViewEndToEnd.java | 26 +-- .../TestMultiDataCenterAdminOperations.java | 26 +-- ...tialUpdateWithActiveActiveReplication.java | 28 +-- .../endToEnd/TestPushJobVersionCleanup.java | 28 +-- ...JobWithEmergencySourceRegionSelection.java | 28 +-- .../TestPushJobWithNativeReplication.java | 27 +-- ...tPushJobWithSourceGridFabricSelection.java | 27 +-- .../endToEnd/TestRocksDBOffsetStore.java | 5 +- .../TestSeparateRealtimeTopicIngestion.java | 28 +-- .../endToEnd/TestStaleDataVisibility.java | 26 +-- .../TestStoreBackupVersionDeletion.java | 27 +-- .../TestStoreGraveyardCleanupService.java | 27 +-- .../venice/endToEnd/TestStoreMigration.java | 29 +-- .../endToEnd/TestStuckConsumerRepair.java | 12 +- .../TestSuperSetSchemaRegistration.java | 5 +- ...TopicWiseSharedConsumerPoolResilience.java | 12 +- .../TestUnusedValueSchemaCleanup.java | 27 +-- .../venice/endToEnd/TestVsonStoreBatch.java | 5 +- .../endToEnd/TestWritePathComputation.java | 10 +- .../RequestBasedMetadataIntegrationTest.java | 12 +- ...ollerKMERegistrationFromMessageHeader.java | 26 +-- ...erverKMERegistrationFromMessageHeader.java | 12 +- .../LeaderFollowerThreadPoolTest.java | 12 +- .../TestRebalanceByDefaultStrategy.java | 19 +- .../IngestionHeartBeatTest.java | 28 +-- .../integration/StorageNodeServiceTest.java | 12 +- .../integration/utils/ServiceFactory.java | 187 +----------------- .../utils/TestFileDescriptorLeak.java | 8 +- .../utils/VeniceClusterWrapper.java | 13 +- .../VeniceMultiClusterCreateOptions.java | 13 ++ .../utils/VeniceMultiClusterWrapper.java | 1 + ...VeniceMultiRegionClusterCreateOptions.java | 13 ++ ...woLayerMultiRegionMultiClusterWrapper.java | 1 + .../venice/restart/TestRestartController.java | 8 +- .../venice/restart/TestRestartRouter.java | 8 +- ...tingSstFilesWithActiveActiveIngestion.java | 25 +-- .../TestRestartServerDuringIngestion.java | 13 +- .../venice/router/ReplicaFailoverTest.java | 8 +- .../venice/router/TestBlobDiscovery.java | 26 +-- ...ConnectionWarmingForApacheAsyncClient.java | 11 +- .../com/linkedin/venice/router/TestRead.java | 12 +- .../venice/router/TestRouterAsyncStart.java | 5 +- .../linkedin/venice/router/TestStreaming.java | 11 +- .../venice/samza/VeniceSystemFactoryTest.java | 5 +- .../venice/server/VeniceServerTest.java | 43 +++- .../ReadComputeValidationTest.java | 11 +- .../storagenode/StorageNodeComputeTest.java | 11 +- .../storagenode/TestEarlyTermination.java | 11 +- .../TestRouterReadQuotaThrottler.java | 12 +- .../DaVinciPartialKeyLookupBenchmark.java | 5 +- .../IngestionBenchmarkInSingleProcess.java | 5 +- .../benchmark/MixedIngestionBenchmark.java | 12 +- .../benchmark/VeniceClientBenchmark.java | 5 +- 100 files changed, 1186 insertions(+), 809 deletions(-) diff --git a/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java b/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java index 49e7ed3a40f..e1ea241a424 100644 --- a/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java +++ b/internal/venice-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java @@ -13,6 +13,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -80,7 +81,16 @@ public VeniceClusterInitializer(String storeName, int routerPort) { Properties clusterConfig = new Properties(); clusterConfig.put(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(ConfigKeys.ROUTER_ENABLE_SSL, false); - this.veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + this.veniceCluster = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); this.veniceCluster.addVeniceServer(new Properties(), serverProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java index ac5c0002893..331870d3335 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceMessageException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.partitioner.DefaultVenicePartitioner; @@ -103,7 +104,9 @@ public abstract class ConsumerIntegrationTest { @BeforeClass public void sharedSetUp() { - cluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); controllerClient = ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java index 8a52c8ae554..6a84c5352a1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java @@ -35,6 +35,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -93,18 +94,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java index 24cb4458c55..81584ae7f01 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java @@ -53,6 +53,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -137,18 +138,18 @@ public void setUp() { CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); serverProperties.put(SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, isAAWCParallelProcessingEnabled()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java index fdd30afd693..fb9008b7711 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java @@ -15,6 +15,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.TestUtils; @@ -23,7 +24,6 @@ import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -51,18 +51,18 @@ public void setUp() { parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false"); parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 2, - 2, - 2, - 2, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(2) + .numberOfChildControllers(2) + .numberOfServers(2) + .numberOfRouters(2) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java index a69bfeb32d2..31e8e18b315 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java @@ -17,6 +17,7 @@ import com.linkedin.venice.helix.HelixReadOnlyLiveClusterConfigRepository; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Version; @@ -47,7 +48,16 @@ public void setUp() { properties.setProperty(LOCAL_REGION_NAME, "dc-0"); properties.setProperty(ALLOW_CLUSTER_WIPE, "true"); properties.setProperty(TOPIC_CLEANUP_DELAY_FACTOR, "0"); - venice = ServiceFactory.getVeniceCluster(1, 1, 1, 1, 100000, false, false, properties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(properties) + .build(); + venice = ServiceFactory.getVeniceCluster(options); clusterName = venice.getClusterName(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java index 6bb682135b3..2027db5b454 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java @@ -15,6 +15,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.utils.TestUtils; @@ -56,18 +57,19 @@ public void setUp() { childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true"); Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.of(childControllerProperties), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .childControllerProperties(childControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java index 47e53e776e2..94cee4cae35 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.NewStoreResponse; import com.linkedin.venice.integration.utils.HelixAsAServiceWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.ZkServerWrapper; @@ -68,7 +69,12 @@ public void setUp() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testClusterResourceInstanceTag() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { String instanceTag = "GENERAL"; String controllerClusterName = "venice-controllers"; @@ -89,7 +95,12 @@ public void testClusterResourceInstanceTag() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testClusterResourceEmptyInstanceTag() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { String instanceTag = ""; String controllerClusterName = "venice-controllers"; @@ -110,7 +121,12 @@ public void testClusterResourceEmptyInstanceTag() { @Test(timeOut = 60 * Time.MS_PER_SECOND) public void testStartHAASHelixControllerAsControllerClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerClusterHAASProperties); waitForNonDeterministicAssertion( @@ -141,7 +157,12 @@ public void testStartHAASHelixControllerAsControllerClusterLeader() { @Test(timeOut = 120 * Time.MS_PER_SECOND) public void testTransitionToHAASControllerAsControllerClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store")); venice.useControllerClient( @@ -182,7 +203,12 @@ public void testTransitionToHAASControllerAsControllerClusterLeader() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testStartHAASControllerAsStorageClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerAndStorageClusterHAASProperties); @@ -212,7 +238,12 @@ public void testStartHAASControllerAsStorageClusterLeader() { @Test(timeOut = 180 * Time.MS_PER_SECOND) public void testTransitionToHAASControllerAsStorageClusterLeader() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store")); @@ -362,7 +393,12 @@ public void testCloudConfig() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testHelixUnknownInstanceOperation() { - try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) { VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerAndStorageClusterHAASProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java index 16fbac94b04..b648d7f1c5f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java @@ -15,6 +15,7 @@ import com.linkedin.venice.helix.HelixCustomizedViewOfflinePushRepository; import com.linkedin.venice.helix.ResourceAssignment; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PartitionAssignment; @@ -42,7 +43,15 @@ public class TestHolisticSeverHealthCheck { @BeforeClass public void setUp() { int numOfController = 1; - cluster = ServiceFactory.getVeniceCluster(numOfController, 2, 1, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); cluster.addVeniceRouter(routerProperties); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java index 8cf6e597dbc..07e4b911e18 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestHybridStoreRepartitioningWithMultiDataCenter.java @@ -10,6 +10,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; import com.linkedin.venice.meta.StoreInfo; @@ -23,7 +24,6 @@ import com.linkedin.venice.utils.Utils; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -51,17 +51,19 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); controllerProps.put(CONTROLLER_ENABLE_HYBRID_STORE_PARTITION_COUNT_UPDATE, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.empty()); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); topicManagers = new ArrayList<>(2); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java index 7db0ecbe789..1698268f7dd 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestParentControllerWithMultiDataCenter.java @@ -22,6 +22,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; import com.linkedin.venice.meta.BufferReplayPolicy; @@ -45,7 +46,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -80,18 +80,20 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); Properties serverProps = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProps), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java index d211d3bfe6e..a9e7e924655 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestStartMultiControllers.java @@ -2,6 +2,7 @@ import com.linkedin.venice.helix.SafeHelixManager; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.utils.HelixUtils; @@ -27,7 +28,11 @@ public class TestStartMultiControllers { @BeforeClass public void setUp() throws Exception { - cluster = ServiceFactory.getVeniceCluster(controllerCount, 0, 0); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(controllerCount) + .numberOfServers(0) + .numberOfRouters(0) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); helixManager = new SafeHelixManager( new ZKHelixManager( cluster.getClusterName(), diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java index 70dbe37c5fd..9b94e735620 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/TestTopicRequestOnHybridDelete.java @@ -17,6 +17,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; @@ -48,7 +49,9 @@ public class TestTopicRequestOnHybridDelete { @BeforeClass public void setUp() { - venice = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + venice = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java index b1303d94c23..2efeaa09834 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/VeniceParentHelixAdminTest.java @@ -82,7 +82,15 @@ public class VeniceParentHelixAdminTest { @BeforeClass public void setUp() { Utils.thisIsLocalhost(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(1, 1, 1, 1, 1, 1); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; venice = multiRegionMultiClusterWrapper.getChildRegions().get(0).getClusters().get(clusterName); } @@ -239,19 +247,20 @@ public void testResourceCleanupCheckForStoreRecreation() { properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(false)); properties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(false)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(properties) + .childControllerProperties(properties); try ( VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(properties), - Optional.of(properties), - Optional.empty()); + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); ControllerClient parentControllerClient = new ControllerClient( twoLayerMultiRegionMultiClusterWrapper.getClusterNames()[0], twoLayerMultiRegionMultiClusterWrapper.getControllerConnectString())) { @@ -433,18 +442,18 @@ public void testSupersetSchemaWithCustomSupersetSchemaGenerator() throws IOExcep properties .put(VeniceControllerWrapper.SUPERSET_SCHEMA_GENERATOR, new SupersetSchemaGeneratorWithCustomProp(CUSTOM_PROP)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(properties); try (VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 0, - 0, - 1, - Optional.of(properties), - Optional.empty(), - Optional.empty())) { + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build())) { String parentControllerUrl = twoLayerMultiRegionMultiClusterWrapper.getControllerConnectString(); try (ControllerClient parentControllerClient = new ControllerClient(twoLayerMultiRegionMultiClusterWrapper.getClusterNames()[0], parentControllerUrl)) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java index 1d52e718f60..f25cc4723ca 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/controller/server/TestAdminSparkServerWithMultiServers.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; @@ -49,7 +50,11 @@ public class TestAdminSparkServerWithMultiServers { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster(1, STORAGE_NODE_COUNT, 0); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(STORAGE_NODE_COUNT) + .numberOfRouters(0) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); controllerClient = ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs()); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java index ae4f6685f4c..c41f4cde894 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/ActiveActiveReplicationForHybridTest.java @@ -56,6 +56,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -141,18 +142,20 @@ public void setUp() { Properties controllerProps = new Properties(); controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java index a997459055e..4ef011f4fc7 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/BlobP2PTransferAmongServersTest.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -294,7 +295,12 @@ public VeniceClusterWrapper initializeVeniceCluster(boolean sameRocksDBFormat) { port2 = TestUtils.getFreePort(); } - VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 0, 0, 2); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); // add first server Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java index 88a3dd3e383..7bbbb60f47b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/CheckSumTest.java @@ -24,6 +24,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.kafka.validation.checksum.CheckSumType; import com.linkedin.venice.meta.PersistenceType; @@ -71,7 +72,16 @@ public void cleanUp() { private VeniceClusterWrapper setUpCluster() { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java index 52fc30e12c9..a45e67dae14 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientRecordTransformerTest.java @@ -33,6 +33,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.utils.DataProviderUtils; @@ -69,7 +70,16 @@ public void setUp() { clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(PUSH_STATUS_STORE_ENABLED, true); clusterConfig.put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 3); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java index e37b7bf4729..4b8b4976ba3 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClientTest.java @@ -70,6 +70,7 @@ import com.linkedin.venice.ingestion.protocol.IngestionStorageMetadata; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.IngestionMetadataUpdateType; @@ -153,7 +154,16 @@ public void setUp() { clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); clusterConfig.put(PUSH_STATUS_STORE_ENABLED, true); clusterConfig.put(DAVINCI_PUSH_STATUS_SCAN_INTERVAL_IN_SECONDS, 3); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java index d8f87545bc4..c82e5a30fdc 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciClusterAgnosticTest.java @@ -25,6 +25,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -40,7 +41,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -78,18 +78,18 @@ public void setUp() { Utils.thisIsLocalhost(); Properties parentControllerProps = new Properties(); parentControllerProps.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 3, - 1, - 3, - Optional.of(parentControllerProps), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(3) + .forkServer(false) + .parentControllerProperties(parentControllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterVenice = multiRegionMultiClusterWrapper.getChildRegions().get(0); clusterNames = multiClusterVenice.getClusterNames(); parentControllerURLs = multiRegionMultiClusterWrapper.getParentControllers() diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java index a15449e50ff..67e6b361a9b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciComputeTest.java @@ -29,6 +29,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -141,7 +142,16 @@ public void setUp() { Utils.thisIsLocalhost(); Properties clusterConfig = new Properties(); clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 1, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java index 3ffd4d21267..7c14a962cb0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DaVinciLiveUpdateSuppressionTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.DaVinciTestContext; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.IngestionMode; import com.linkedin.venice.meta.StoreInfo; @@ -66,7 +67,16 @@ public void setUp() { Utils.thisIsLocalhost(); Properties clusterConfig = new Properties(); clusterConfig.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - cluster = ServiceFactory.getVeniceCluster(1, 2, 1, 1, 100, false, false, clusterConfig); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(clusterConfig) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); d2Client = new D2ClientBuilder().setZkHosts(cluster.getZk().getAddress()) .setZkSessionTimeout(3, TimeUnit.SECONDS) .setZkStartupTimeout(3, TimeUnit.SECONDS) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java index 89f0a0bc775..6eb358ff546 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/DataRecoveryTest.java @@ -40,6 +40,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.Version; @@ -98,18 +99,20 @@ public void setUp() { controllerProps.put(ALLOW_CLUSTER_WIPE, "true"); controllerProps.put(TOPIC_CLEANUP_SLEEP_INTERVAL_BETWEEN_TOPIC_LIST_FETCH_MS, "1000"); controllerProps.put(MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE, "0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); clusterName = CLUSTER_NAMES[0]; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java index f60a2eb93d9..4fb6674b2f1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/NearlineE2ELatencyTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -70,18 +71,18 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); multiRegionMultiClusterWrapper.logMultiCluster(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java index e4f80c5d116..b1b6c44311c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/OneTouchDataRecoveryTest.java @@ -13,6 +13,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.RegionPushDetails; import com.linkedin.venice.meta.Version; @@ -58,18 +59,18 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java index 93d3e0cb06e..d3806827051 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateClusterConfigTest.java @@ -14,6 +14,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.StoreInfo; @@ -21,7 +22,6 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.Assert; @@ -46,18 +46,20 @@ public void setUp() { controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); controllerProps.put(ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, true); controllerProps.put(ConfigKeys.ENABLE_PARTIAL_UPDATE_FOR_HYBRID_NON_ACTIVE_ACTIVE_USER_STORES, true); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java index ab31cccda4f..6303037666c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PartialUpdateTest.java @@ -74,6 +74,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.BackupStrategy; @@ -115,7 +116,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -175,18 +175,20 @@ public void setUp() { Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); controllerProps.put(ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java index 078a5a51631..7bc8c35ca85 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushJobDetailsTest.java @@ -39,6 +39,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.status.PushJobDetailsStatus; @@ -58,7 +59,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -100,18 +100,19 @@ private void setUp(boolean useCustomCheckpoints) throws IOException { // Need to add this in controller props when creating venice system for tests parentControllerProperties.setProperty(ConfigKeys.PUSH_JOB_STATUS_STORE_CLUSTER_NAME, "venice-cluster0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); String clusterName = multiRegionMultiClusterWrapper.getClusterNames()[0]; VeniceMultiClusterWrapper childRegionMultiClusterWrapper = multiRegionMultiClusterWrapper.getChildRegions().get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java index 1ebca163050..ad8a8f4fb86 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/PushStatusStoreMultiColoTest.java @@ -25,6 +25,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -34,7 +35,6 @@ import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.testng.annotations.AfterClass; @@ -76,18 +76,19 @@ public void setUp() { extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, String.valueOf(true)); extraProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, String.valueOf(true)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - NUMBER_OF_PARENT_CONTROLLERS, - NUMBER_OF_CHILD_CONTROLLERS, - NUMBER_OF_SERVERS, - NUMBER_OF_ROUTERS, - REPLICATION_FACTOR, - Optional.of(extraProperties), - Optional.of(extraProperties), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(NUMBER_OF_PARENT_CONTROLLERS) + .numberOfChildControllers(NUMBER_OF_CHILD_CONTROLLERS) + .numberOfServers(NUMBER_OF_SERVERS) + .numberOfRouters(NUMBER_OF_ROUTERS) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(extraProperties) + .childControllerProperties(extraProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java index 4828c30af99..7cdf1d51142 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/RocksDBPlainTableTest.java @@ -9,6 +9,7 @@ import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.utils.Time; @@ -25,7 +26,9 @@ public class RocksDBPlainTableTest { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java index fe24d002e2d..7a8cd0e5ad6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/StoreMetadataRecoveryTest.java @@ -21,6 +21,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.TestUtils; @@ -62,18 +63,18 @@ public void setUp() { serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); // 1 parent controller, 1 child region, 1 clusters per child region, 2 servers per cluster - twoLayerClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 2, - 0, - 2, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + twoLayerClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterWrapper = twoLayerClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java index a92865751ac..c435f74c7ef 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveIngestion.java @@ -37,6 +37,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -126,18 +127,20 @@ public void setUp() { serverProperties.put(SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, isAAWCParallelProcessingEnabled()); Properties controllerProps = new Properties(); controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 20); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java index 63fb50cbaf5..fc53acfe2c8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationForIncPush.java @@ -30,6 +30,7 @@ import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.HybridStoreConfig; @@ -103,18 +104,20 @@ public void setUp() { controllerProps.put(ENABLE_ACTIVE_ACTIVE_REPLICATION_AS_DEFAULT_FOR_HYBRID_STORE, true); controllerProps.put(ENABLE_INCREMENTAL_PUSH_FOR_HYBRID_ACTIVE_ACTIVE_USER_STORES, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); clusterName = this.clusterNames[0]; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java index d3f50175547..aba05953103 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDelayedLeaderPromotion.java @@ -22,6 +22,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -62,18 +63,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java index 534490697e8..93bd5c4ffda 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestActiveActiveReplicationWithDownRegion.java @@ -22,6 +22,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.samza.VeniceSystemProducer; @@ -87,18 +88,20 @@ public void setUp() { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java index 9b3ab2ff9cb..a803a10f324 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBackupVersionDatabaseOptimization.java @@ -20,6 +20,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.utils.TestUtils; @@ -52,7 +53,16 @@ public void setUp() { extraProperties.setProperty(SERVER_OPTIMIZE_DATABASE_SERVICE_SCHEDULE_INTERNAL_SECONDS, "1"); extraProperties.setProperty(SERVER_OPTIMIZE_DATABASE_FOR_BACKUP_VERSION_NO_READ_THRESHOLD_SECONDS, "3"); - venice = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + venice = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java index 4015671b616..61852c7ae47 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchForRocksDB.java @@ -11,6 +11,7 @@ import static com.linkedin.venice.ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import java.util.Properties; @@ -21,7 +22,9 @@ public class TestBatchForRocksDB extends TestBatch { @Override public VeniceClusterWrapper initializeVeniceCluster() { - VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 0, 0); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); Properties serverProperties = new Properties(); serverProperties.put(PERSISTENCE_TYPE, PersistenceType.ROCKS_DB); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java index cef9e24a1e5..2e01a0a0f1a 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestBatchReportIncrementalPush.java @@ -38,6 +38,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -53,7 +54,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -79,18 +79,20 @@ public void setUp() { serverProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); serverProperties.setProperty(SERVER_BATCH_REPORT_END_OF_INCREMENTAL_PUSH_STATUS_ENABLED, Long.toString(60L)); Properties controllerProps = new Properties(); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java index b71eafc8f8e..cdc9d3edaa9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDeferredVersionSwap.java @@ -10,6 +10,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreInfo; import com.linkedin.venice.meta.Version; @@ -21,7 +22,6 @@ import java.io.File; import java.io.IOException; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -47,17 +47,20 @@ public void setUp() { controllerProps.put(CONTROLLER_DEFERRED_VERSION_SWAP_SERVICE_ENABLED, true); Properties serverProperties = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java index a5a67cd9fc0..4a461e63d11 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestDumpIngestionContext.java @@ -20,6 +20,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -28,7 +29,6 @@ import com.linkedin.venice.utils.Utils; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import org.apache.avro.Schema; @@ -62,18 +62,20 @@ public void setUp() { KafkaConsumerServiceDelegator.ConsumerPoolStrategyType.CURRENT_VERSION_PRIORITIZATION.name()); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); this.parentController = parentControllers.get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java index cdd919d02cd..377acd03fc2 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestFatalDataValidationExceptionHandling.java @@ -37,6 +37,7 @@ import com.linkedin.venice.guid.GuidUtils; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.Version; @@ -94,7 +95,16 @@ private VeniceClusterWrapper setUpCluster() { extraProperties .setProperty(FATAL_DATA_VALIDATION_FAILURE_TOPIC_RETENTION_MS, Long.toString(TimeUnit.SECONDS.toMillis(30))); extraProperties.setProperty(MIN_NUMBER_OF_UNUSED_KAFKA_TOPICS_TO_PRESERVE, "0"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java index e5a85896c08..507f4d3f1da 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHelixCustomizedView.java @@ -11,6 +11,7 @@ import com.linkedin.venice.helix.VeniceOfflinePushMonitorAccessor; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -80,7 +81,15 @@ public class TestHelixCustomizedView { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 0, replicationFactor, 10, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(replicationFactor) + .partitionSize(10) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); admin = new ZKHelixAdmin(veniceCluster.getZk().getAddress()); Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java index 9cf804fd5ea..f4a22b119e0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybrid.java @@ -56,6 +56,7 @@ import com.linkedin.venice.helix.HelixBaseRoutingRepository; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.kafka.protocol.GUID; @@ -475,8 +476,16 @@ public void testSamzaBatchLoad() throws Exception { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); SystemProducer veniceBatchProducer = null; - try (VeniceClusterWrapper veniceClusterWrapper = - ServiceFactory.getVeniceCluster(1, 3, 1, 2, 1000000, false, false, extraProperties)) { + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options)) { try { Admin admin = veniceClusterWrapper.getLeaderVeniceController().getVeniceAdmin(); String clusterName = veniceClusterWrapper.getClusterName(); @@ -725,9 +734,16 @@ public void testMultiStreamReprocessingSystemProducers() { public void testLeaderHonorLastTopicSwitchMessage() throws Exception { Properties extraProperties = new Properties(); extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(10L)); - try ( - VeniceClusterWrapper venice = - ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options); ControllerClient controllerClient = new ControllerClient(venice.getClusterName(), venice.getAllControllersURLs())) { long streamingRewindSeconds = 25L; @@ -1606,8 +1622,16 @@ public void testLeaderShouldCalculateRewindDuringPromotion() { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(20L)); final int partitionCount = 1; final int keyCount = 10; - try (VeniceClusterWrapper cluster = - ServiceFactory.getVeniceCluster(1, 1, 1, 1, 1000000, false, false, extraProperties)) { + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { UpdateStoreQueryParams params = new UpdateStoreQueryParams() // set hybridRewindSecond to a big number so following versions won't ignore old records in RT .setHybridRewindSeconds(10) @@ -1695,7 +1719,16 @@ private static Pair getKafkaKeyAndValueEnvelope( private static VeniceClusterWrapper setUpCluster(boolean enablePartitionWiseSharedConsumer) { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java index 9d9fae8396b..daca19c9a4c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridMultiRegion.java @@ -28,6 +28,7 @@ import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.DataReplicationPolicy; import com.linkedin.venice.meta.HybridStoreConfig; @@ -339,19 +340,20 @@ private static VeniceTwoLayerMultiRegionMultiClusterWrapper setUpCluster() { serverProperties.setProperty(SERVER_CONSUMER_POOL_SIZE_PER_KAFKA_CLUSTER, "3"); serverProperties.setProperty(SERVER_DEDICATED_DRAINER_FOR_SORTED_INPUT_ENABLED, "true"); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(Optional.of(parentControllerProps).orElse(null)) + .childControllerProperties(Optional.of(childControllerProperties).orElse(null)) + .serverProperties(serverProperties); VeniceTwoLayerMultiRegionMultiClusterWrapper cluster = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 1, - 1, - 1, - 2, - 1, - 1, - Optional.of(parentControllerProps), - Optional.of(childControllerProperties), - Optional.of(serverProperties), - false); + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); return cluster; } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java index e573d112f5d..763f9853342 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestHybridQuota.java @@ -28,6 +28,7 @@ import com.linkedin.venice.helix.SafeHelixManager; import com.linkedin.venice.helix.ZkClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.PersistenceType; import com.linkedin.venice.meta.Store; @@ -73,7 +74,16 @@ public void setUp() { extraProperties.setProperty(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1L)); // N.B.: RF 2 with 3 servers is important, in order to test both the leader and follower code paths - sharedVenice = ServiceFactory.getVeniceCluster(1, 0, 0, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + sharedVenice = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); routerProperties.put(HELIX_HYBRID_STORE_QUOTA_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java index 31fa014783b..01de56380e5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestLeaderReplicaFailover.java @@ -23,6 +23,7 @@ import com.linkedin.venice.helix.HelixBaseRoutingRepository; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Instance; @@ -92,8 +93,17 @@ public void setUp() { int numberOfServer = 3; int numberOfRouter = 1; - clusterWrapper = ServiceFactory - .getVeniceCluster(numberOfController, numberOfServer, numberOfRouter, 3, 1, false, false, serverProperties); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .replicationFactor(3) + .partitionSize(1) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(serverProperties) + .build(); + clusterWrapper = ServiceFactory.getVeniceCluster(options); clusterName = clusterWrapper.getClusterName(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java index 2e30a113a01..2f0fd279ad0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMaterializedViewEndToEnd.java @@ -30,6 +30,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.MaterializedViewParameters; @@ -54,7 +55,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -84,18 +84,18 @@ public void setUp() { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 2, - 1, - 1, - 1, - 2, - 1, - 2, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(2) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java index 67fd2815614..d469b2c726f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestMultiDataCenterAdminOperations.java @@ -16,6 +16,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -26,7 +27,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -60,18 +60,18 @@ public class TestMultiDataCenterAdminOperations { public void setUp() { Properties serverProperties = new Properties(); serverProperties.setProperty(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childClusters = multiRegionMultiClusterWrapper.getChildRegions(); childControllers = childClusters.stream() diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java index aa60a5116cb..1f93b67b518 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPartialUpdateWithActiveActiveReplication.java @@ -24,6 +24,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.samza.VeniceObjectWithTimestamp; @@ -42,7 +43,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -112,18 +112,20 @@ public void setUp() throws IOException { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(PARENT_KAFKA_CLUSTER_FABRIC_LIST, DEFAULT_PARENT_DATA_CENTER_REGION_NAME); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java index b945dd5f484..7502b106dfa 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobVersionCleanup.java @@ -14,6 +14,7 @@ import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.server.VeniceServer; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -22,7 +23,6 @@ import com.linkedin.venice.utils.Utils; import java.io.File; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -54,18 +54,20 @@ public void setUp() { controllerProps.put(ADMIN_HELIX_MESSAGING_CHANNEL_ENABLED, false); controllerProps.put(PARTICIPANT_MESSAGE_STORE_ENABLED, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java index a4113f88b79..a684df224fb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithEmergencySourceRegionSelection.java @@ -21,6 +21,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -29,7 +30,6 @@ import com.linkedin.venice.utils.Utils; import java.io.File; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -85,18 +85,20 @@ public void setUp() { controllerProps.put(NATIVE_REPLICATION_SOURCE_FABRIC, "dc-0"); controllerProps.put(EMERGENCY_SOURCE_REGION, "dc-2"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java index 6a4fcc4d962..c7edff172a9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithNativeReplication.java @@ -68,6 +68,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -176,18 +177,20 @@ public void setUp() { controllerProps.put(PUSH_JOB_STATUS_STORE_CLUSTER_NAME, SYSTEM_STORE_CLUSTER); controllerProps.put(EMERGENCY_SOURCE_REGION, "dc-0"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); VeniceClusterWrapper clusterWrapper = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java index 08f1862b5e6..c343eaabf27 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestPushJobWithSourceGridFabricSelection.java @@ -19,6 +19,7 @@ import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.VeniceUserStoreType; @@ -67,18 +68,20 @@ public void setUp() { serverProperties.setProperty(SERVER_DATABASE_SYNC_BYTES_INTERNAL_FOR_DEFERRED_WRITE_MODE, "300"); Properties controllerProps = new Properties(); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllerRegionName = multiRegionMultiClusterWrapper.getParentRegionName() + ".parent"; clusterNames = multiRegionMultiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java index f6b9d61bdc2..6919506af51 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestRocksDBOffsetStore.java @@ -7,6 +7,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -26,7 +27,9 @@ public class TestRocksDBOffsetStore { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java index 9656e6ebffb..7baed5aef98 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSeparateRealtimeTopicIngestion.java @@ -40,6 +40,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.ReadOnlySchemaRepository; @@ -63,7 +64,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -103,18 +103,20 @@ public void setUp() { serverProperties.put(ConfigKeys.SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, Boolean.toString(false)); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 2, - 1, - REPLICATION_FACTOR, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(REPLICATION_FACTOR) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java index eab70f0733e..dd08d44cb2e 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStaleDataVisibility.java @@ -14,6 +14,7 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreDataAudit; import com.linkedin.venice.meta.StoreInfo; @@ -26,7 +27,6 @@ import java.io.File; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -56,18 +56,18 @@ public class TestStaleDataVisibility { public void setUp() { Properties serverProperties = new Properties(); serverProperties.setProperty(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, Long.toString(1)); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childClusters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java index a988b7488f3..280d365b483 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreBackupVersionDeletion.java @@ -17,6 +17,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; import com.linkedin.venice.meta.Version; @@ -27,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -60,17 +60,20 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); StoreBackupVersionCleanupService.setWaitTimeDeleteRepushSourceVersion(10); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); veniceHelixAdmin = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java index 1e65af29e15..9eb2d1bcec4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreGraveyardCleanupService.java @@ -13,12 +13,12 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.StoreGraveyard; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -49,18 +49,19 @@ public void setUp() { parentControllerProperties.put(CONTROLLER_STORE_GRAVEYARD_CLEANUP_SLEEP_INTERVAL_BETWEEN_LIST_FETCH_MINUTES, 0); parentControllerProperties.put(CONTROLLER_STORE_GRAVEYARD_CLEANUP_DELAY_MINUTES, -1); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java index 6c257103430..e52b9581f4c 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStoreMigration.java @@ -54,6 +54,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Store; @@ -133,19 +134,21 @@ public void setUp() throws Exception { // 1 parent controller, 1 child region, 2 clusters per child region, 2 servers per cluster // RF=2 to test both leader and follower SNs - twoLayerMultiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 2, - 1, - 2, - Optional.of(parentControllerProperties), - Optional.empty(), - Optional.of(serverProperties), - false, - true); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .sslToStorageNodes(true) + .forkServer(false) + .parentControllerProperties(parentControllerProperties) + .childControllerProperties(null) + .serverProperties(serverProperties); + twoLayerMultiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterWrapper = twoLayerMultiRegionMultiClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterWrapper.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java index 50d83ae91dc..2afa701ae5b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestStuckConsumerRepair.java @@ -37,6 +37,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -79,7 +80,16 @@ public void setUp() { private static VeniceClusterWrapper setUpCluster() { Properties extraProperties = new Properties(); extraProperties.setProperty(DEFAULT_MAX_NUMBER_OF_PARTITIONS, "5"); - VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 1, 1, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); // Add Venice Router Properties routerProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java index 7259546355c..1bb4b90e541 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestSuperSetSchemaRegistration.java @@ -9,6 +9,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -83,7 +84,9 @@ private static Schema writeComplicatedAvroFileWithUserSchema( @BeforeClass public void setUp() { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(); // Now with SSL! + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Now with SSL! } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java index 9ad1201ca8a..bdea83ce042 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestTopicWiseSharedConsumerPoolResilience.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.hadoop.VenicePushJob; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.meta.Version; @@ -54,7 +55,16 @@ public void setUp() { extraProperties.setProperty( SERVER_SHARED_CONSUMER_ASSIGNMENT_STRATEGY, KafkaConsumerService.ConsumerAssignmentStrategy.TOPIC_WISE_SHARED_CONSUMER_ASSIGNMENT_STRATEGY.name()); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 1000000, false, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(1000000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java index 9d166ec95d5..c92cf664f4b 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestUnusedValueSchemaCleanup.java @@ -18,6 +18,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.utils.IntegrationTestPushUtils; @@ -27,7 +28,6 @@ import java.io.File; import java.io.IOException; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -62,17 +62,20 @@ public void setUp() { Properties serverProperties = new Properties(); serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties)); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); veniceHelixAdmin = diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java index 16f97ddb984..e143bcef08f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestVsonStoreBatch.java @@ -25,6 +25,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.schema.vson.VsonAvroSchemaAdapter; @@ -69,7 +70,9 @@ public class TestVsonStoreBatch { @BeforeClass public void setUp() { - veniceCluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); controllerClient = new ControllerClient( veniceCluster.getClusterName(), veniceCluster.getLeaderVeniceController().getControllerUrl()); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java index 60da54bae1f..2b0bbcbee17 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/endToEnd/TestWritePathComputation.java @@ -11,6 +11,7 @@ import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Time; @@ -72,8 +73,15 @@ public void testFeatureFlagSingleDC() { @Test(timeOut = 90 * Time.MS_PER_SECOND) public void testFeatureFlagMultipleDC() { + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(1) + .numberOfRouters(0); try (VeniceTwoLayerMultiRegionMultiClusterWrapper twoLayerMultiRegionMultiClusterWrapper = - ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(1, 1, 1, 1, 1, 0)) { + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build())) { VeniceMultiClusterWrapper multiCluster = twoLayerMultiRegionMultiClusterWrapper.getChildRegions().get(0); VeniceControllerWrapper parentController = twoLayerMultiRegionMultiClusterWrapper.getParentControllers().get(0); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java index 79a63e11380..ae20ae15af9 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/fastclient/meta/RequestBasedMetadataIntegrationTest.java @@ -15,6 +15,7 @@ import com.linkedin.venice.fastclient.utils.ClientTestUtils; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.OnlineInstanceFinder; @@ -61,7 +62,16 @@ public void setUp() throws Exception { Utils.thisIsLocalhost(); Properties props = new Properties(); props.put(SERVER_HTTP2_INBOUND_ENABLED, "true"); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 1, 2, 100, true, false, props); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .extraProperties(props) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); r2Client = ClientTestUtils.getR2Client(); d2Client = D2TestUtils.getAndStartHttpsD2Client(veniceCluster.getZk().getAddress()); storeName = veniceCluster.createStore(KEY_COUNT); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java index c7c0388affd..817e7af0439 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestControllerKMERegistrationFromMessageHeader.java @@ -8,13 +8,13 @@ import com.linkedin.venice.integration.utils.ServiceFactory; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.serialization.avro.AvroProtocolDefinition; import com.linkedin.venice.serialization.avro.KafkaValueSerializer; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; import java.util.List; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.TimeUnit; import java.util.stream.IntStream; @@ -45,17 +45,19 @@ public void setUp() { controllerProps.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, 3); controllerProps.put(DEFAULT_PARTITION_SIZE, 1024); controllerProps.put(KME_REGISTRATION_FROM_MESSAGE_HEADER_ENABLED, true); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 3, - 1, - 1, - 1, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.empty()); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(3) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); } diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java index 7009d74ef8e..5384fa538c1 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helix/TestServerKMERegistrationFromMessageHeader.java @@ -6,6 +6,7 @@ import com.linkedin.venice.controllerapi.ControllerClient; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -52,8 +53,15 @@ public class TestServerKMERegistrationFromMessageHeader { @BeforeClass public void setUp() { - cluster = - ServiceFactory.getVeniceCluster(numOfController, 0, numOfRouters, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(0) + .numberOfRouters(numOfRouters) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); clusterName = cluster.getClusterName(); Properties serverProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java index 50b23b5d4d8..8dd759068a4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/LeaderFollowerThreadPoolTest.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -45,8 +46,15 @@ public void setUp() { int numOfController = 1; int numOfServers = 0; int numOfRouters = 1; - cluster = ServiceFactory - .getVeniceCluster(numOfController, numOfServers, numOfRouters, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController) + .numberOfServers(numOfServers) + .numberOfRouters(numOfRouters) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterMethod diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java index 8bb5bca6187..7100ab88514 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/helixrebalance/TestRebalanceByDefaultStrategy.java @@ -6,6 +6,7 @@ import com.linkedin.venice.helix.HelixState; import com.linkedin.venice.helix.Replica; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; import com.linkedin.venice.utils.TestUtils; @@ -47,14 +48,16 @@ public class TestRebalanceByDefaultStrategy { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster( - numberOfController, - numberOfServer, - numberOfRouter, - replicationFactor, - partitionSize, - false, - false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .replicationFactor(replicationFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); String storeName = Utils.getUniqueString("testRollingUpgrade"); long storageQuota = (long) partitionSize * partitionNumber; cluster.getNewStore(storeName); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java index bc7cee1128b..2e10e3dea20 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/ingestionHeartbeat/IngestionHeartBeatTest.java @@ -31,6 +31,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope; import com.linkedin.venice.message.KafkaKey; @@ -58,7 +59,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -90,18 +90,20 @@ public void setUp() { Properties serverProperties = new Properties(); Properties controllerProps = new Properties(); controllerProps.put(ConfigKeys.CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, false); - this.multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_CHILD_DATACENTERS, - NUMBER_OF_CLUSTERS, - 1, - 1, - 4, - 1, - 2, - Optional.of(controllerProps), - Optional.of(controllerProps), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS) + .numberOfClusters(NUMBER_OF_CLUSTERS) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(4) + .numberOfRouters(1) + .replicationFactor(2) + .forkServer(false) + .parentControllerProperties(controllerProps) + .childControllerProperties(controllerProps) + .serverProperties(serverProperties); + this.multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); this.childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); if (parentControllers.size() != 1) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java index 0e7295e9987..ebc0a2d9188 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/StorageNodeServiceTest.java @@ -2,6 +2,7 @@ import com.linkedin.venice.httpclient.HttpClientUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.utils.SslUtils; @@ -21,11 +22,18 @@ public class StorageNodeServiceTest { @Test public void storageServerRespondsToRequests() throws ExecutionException, InterruptedException, IOException { Utils.thisIsLocalhost(); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(1) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); try ( CloseableHttpAsyncClient client = HttpClientUtils.getMinimalHttpClient(1, 1, Optional.of(SslUtils.getVeniceLocalSslFactory())); - VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(1, 1, 0, 1, 100, true, false)) { - + VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options)) { client.start(); VeniceServerWrapper sslServer = venice.getVeniceServers().get(0); // This should work, talking ssl to an ssl storage node diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java index 6a51152ef6e..fa7b59b7dc5 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/ServiceFactory.java @@ -281,7 +281,7 @@ public static MockHttpServerWrapper getMockHttpServer(String serviceName) { /** * Start up a testing Venice cluster in another process. * - * The reason to call this method instead of other {@link #getVeniceCluster()} methods is + * The reason to call this method instead of other {@link #getVeniceCluster(VeniceClusterCreateOptions)} methods is * when one wants to maximize its testing environment isolation. * Example usage: {@literal com.linkedin.venice.benchmark.IngestionBenchmarkWithTwoProcesses} * @@ -294,7 +294,7 @@ public static void startVeniceClusterInAnotherProcess(String clusterInfoFilePath /** * Start up a testing Venice cluster in another process. * - * The reason to call this method instead of other {@link #getVeniceCluster()} methods is + * The reason to call this method instead of other {@link #getVeniceCluster(VeniceClusterCreateOptions)} methods is * when one wants to maximize its testing environment isolation. * Example usage: {@literal com.linkedin.venice.benchmark.IngestionBenchmarkWithTwoProcesses} * @@ -319,193 +319,10 @@ public static VeniceClusterWrapper getVeniceCluster(VeniceClusterCreateOptions o return getService(VeniceClusterWrapper.SERVICE_NAME, VeniceClusterWrapper.generateService(options)); } - /** - * @deprecated - *

Use {@link ServiceFactory#getVeniceCluster(VeniceClusterCreateOptions)} instead. - */ - @Deprecated - public static VeniceClusterWrapper getVeniceCluster() { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - int partitionSize, - boolean sslToStorageNodes, - boolean sslToKafka, - Properties extraProperties) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .partitionSize(partitionSize) - .sslToStorageNodes(sslToStorageNodes) - .sslToKafka(sslToKafka) - .extraProperties(extraProperties) - .build(); - return getVeniceCluster(options); - } - - @Deprecated - public static VeniceClusterWrapper getVeniceCluster( - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - int partitionSize, - boolean sslToStorageNodes, - boolean sslToKafka) { - VeniceClusterCreateOptions options = - new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .partitionSize(partitionSize) - .sslToStorageNodes(sslToStorageNodes) - .sslToKafka(sslToKafka) - .build(); - return getVeniceCluster(options); - } - public static VeniceMultiClusterWrapper getVeniceMultiClusterWrapper(VeniceMultiClusterCreateOptions options) { return getService(VeniceMultiClusterWrapper.SERVICE_NAME, VeniceMultiClusterWrapper.generateService(options)); } - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps) { - return getVeniceTwoLayerMultiRegionMultiClusterWrapper( - numberOfRegions, - numberOfClustersInEachRegion, - numberOfParentControllers, - numberOfControllers, - numberOfServers, - numberOfRouters, - replicationFactor, - parentControllerProps, - childControllerProperties, - serverProps, - false); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps, - boolean forkServer, - boolean sslToStorageNodes) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .sslToStorageNodes(sslToStorageNodes) - .forkServer(forkServer); - - parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); - childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); - serverProps.ifPresent(optionsBuilder::serverProperties); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( - int numberOfRegions, - int numberOfClustersInEachRegion, - int numberOfParentControllers, - int numberOfControllers, - int numberOfServers, - int numberOfRouters, - int replicationFactor, - Optional parentControllerProps, - Optional childControllerProperties, - Optional serverProps, - boolean forkServer) { - VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = - new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(numberOfRegions) - .numberOfClusters(numberOfClustersInEachRegion) - .numberOfParentControllers(numberOfParentControllers) - .numberOfChildControllers(numberOfControllers) - .numberOfServers(numberOfServers) - .numberOfRouters(numberOfRouters) - .replicationFactor(replicationFactor) - .forkServer(forkServer); - - parentControllerProps.ifPresent(optionsBuilder::parentControllerProperties); - childControllerProperties.ifPresent(optionsBuilder::childControllerProperties); - serverProps.ifPresent(optionsBuilder::serverProperties); - return getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); - } - public static VeniceTwoLayerMultiRegionMultiClusterWrapper getVeniceTwoLayerMultiRegionMultiClusterWrapper( VeniceMultiRegionClusterCreateOptions options) { return getService( diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java index c32c4419e38..9e70e59d808 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/TestFileDescriptorLeak.java @@ -37,7 +37,9 @@ public void testKafkaBrokerLeak() { @Test(invocationCount = 20, groups = { "flaky" }) public void testVeniceClusterLeak() { - try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster()) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + try (VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options)) { LOGGER.info("Created VeniceClusterWrapper: {}", veniceClusterWrapper.getClusterName()); } } @@ -46,7 +48,9 @@ public void testVeniceClusterLeak() { @BeforeClass public void setUpReusableCluster() { - this.veniceClusterWrapper = ServiceFactory.getVeniceCluster(1, 2, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(2).numberOfRouters(1).build(); + this.veniceClusterWrapper = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java index e1a28520f71..198ae0bf0bf 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceClusterWrapper.java @@ -589,6 +589,7 @@ public VeniceControllerWrapper addVeniceController(Properties properties) { .clusterToD2(clusterToD2) .clusterToServerD2(clusterToServerD2) .extraProperties(properties) + .dynamicAccessController(options.getAccessController()) .build()); synchronized (this) { veniceControllerWrappers.put(veniceControllerWrapper.getPort(), veniceControllerWrapper); @@ -1172,8 +1173,16 @@ public static void main(String[] args) throws IOException { Utils.thisIsLocalhost(); Properties extraProperties = new Properties(); extraProperties.put(DEFAULT_MAX_NUMBER_OF_PARTITIONS, numberOfPartitions); - VeniceClusterWrapper veniceClusterWrapper = - ServiceFactory.getVeniceCluster(1, 1, 1, 1, 10 * 1024 * 1024, false, false, extraProperties); + VeniceClusterCreateOptions options1 = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(1) + .partitionSize(10 * 1024 * 1024) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + VeniceClusterWrapper veniceClusterWrapper = ServiceFactory.getVeniceCluster(options1); String storeName = Utils.getUniqueString("storeForMainMethodOf" + VeniceClusterWrapper.class.getSimpleName()); String controllerUrl = veniceClusterWrapper.getRandomVeniceController().getControllerUrl(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java index d02a19a1b45..e32457bb6eb 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterCreateOptions.java @@ -10,6 +10,7 @@ import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.STANDALONE_REGION_NAME; +import com.linkedin.venice.acl.DynamicAccessController; import java.util.Collections; import java.util.Map; import java.util.Properties; @@ -37,6 +38,7 @@ public class VeniceMultiClusterCreateOptions { private final PubSubBrokerWrapper pubSubBrokerWrapper; private final Properties childControllerProperties; private final Properties extraProperties; + private final DynamicAccessController accessController; public String getRegionName() { return regionName; @@ -122,6 +124,10 @@ public Properties getExtraProperties() { return extraProperties; } + public DynamicAccessController getAccessController() { + return accessController; + } + @Override public String toString() { return new StringBuilder().append("VeniceMultiClusterCreateOptions - ") @@ -212,6 +218,7 @@ private VeniceMultiClusterCreateOptions(Builder builder) { extraProperties = builder.extraProperties; forkServer = builder.forkServer; kafkaClusterMap = builder.kafkaClusterMap; + accessController = builder.accessController; } public static class Builder { @@ -236,6 +243,7 @@ public static class Builder { private PubSubBrokerWrapper pubSubBrokerWrapper; private Properties childControllerProperties; private Properties extraProperties; + private DynamicAccessController accessController; public Builder numberOfClusters(int numberOfClusters) { this.numberOfClusters = numberOfClusters; @@ -346,6 +354,11 @@ public Builder extraProperties(Properties extraProperties) { return this; } + public Builder accessController(DynamicAccessController accessController) { + this.accessController = accessController; + return this; + } + private void addDefaults() { if (numberOfClusters == 0) { numberOfClusters = 1; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java index 88baea83932..21e325b1d92 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiClusterWrapper.java @@ -117,6 +117,7 @@ static ServiceProvider generateService(VeniceMultiClu .clusterToServerD2(clusterToServerD2) .sslToKafka(false) .d2Enabled(true) + .dynamicAccessController(options.getAccessController()) .extraProperties(controllerProperties) .build(); for (int i = 0; i < options.getNumberOfControllers(); i++) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java index c2dafc85268..08b080d654a 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceMultiRegionClusterCreateOptions.java @@ -7,6 +7,7 @@ import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_KAFKA; import static com.linkedin.venice.integration.utils.VeniceClusterWrapperConstants.DEFAULT_SSL_TO_STORAGE_NODES; +import com.linkedin.venice.acl.DynamicAccessController; import com.linkedin.venice.authorization.AuthorizerService; import java.util.Properties; @@ -28,6 +29,7 @@ public class VeniceMultiRegionClusterCreateOptions { private final AuthorizerService parentAuthorizerService; private final String parentVeniceZkBasePath; private final String childVeniceZkBasePath; + private final DynamicAccessController accessController; public int getNumberOfRegions() { return numberOfRegions; @@ -93,6 +95,10 @@ public String getChildVeniceZkBasePath() { return childVeniceZkBasePath; } + public DynamicAccessController getAccessController() { + return accessController; + } + @Override public String toString() { return new StringBuilder().append("VeniceMultiClusterCreateOptions - ") @@ -163,6 +169,7 @@ private VeniceMultiRegionClusterCreateOptions(Builder builder) { parentAuthorizerService = builder.parentAuthorizerService; parentVeniceZkBasePath = builder.parentVeniceZkBasePath; childVeniceZkBasePath = builder.childVeniceZkBasePath; + accessController = builder.accessController; } public static class Builder { @@ -182,6 +189,7 @@ public static class Builder { private AuthorizerService parentAuthorizerService; private String parentVeniceZkBasePath = "/"; private String childVeniceZkBasePath = "/"; + private DynamicAccessController accessController; public Builder numberOfRegions(int numberOfRegions) { this.numberOfRegions = numberOfRegions; @@ -271,6 +279,11 @@ public Builder childVeniceZkBasePath(String veniceZkBasePath) { return this; } + public Builder accessController(DynamicAccessController accessController) { + this.accessController = accessController; + return this; + } + private void addDefaults() { if (numberOfRegions == 0) { numberOfRegions = 1; diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java index 547c3db34a2..4d09daf5861 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/integration/utils/VeniceTwoLayerMultiRegionMultiClusterWrapper.java @@ -224,6 +224,7 @@ static ServiceProvider generateSer .clusterToServerD2(clusterToServerD2) .regionName(parentRegionName) .authorizerService(options.getParentAuthorizerService()) + .dynamicAccessController(options.getAccessController()) .build(); // Create parentControllers for multi-cluster for (int i = 0; i < options.getNumberOfParentControllers(); i++) { diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java index 286de4b2c01..179f5bcadae 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartController.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.pushmonitor.ExecutionStatus; @@ -31,7 +32,12 @@ public void setUp() { int numberOfServer = 1; int numberOfRouter = 1; - cluster = ServiceFactory.getVeniceCluster(numberOfController, numberOfServer, numberOfRouter); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterMethod diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java index bc48bffbccd..495c28e9110 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartRouter.java @@ -5,6 +5,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.RoutingDataRepository; @@ -30,7 +31,12 @@ public void setUp() { int numberOfServer = 1; int numberOfRouter = 2; - cluster = ServiceFactory.getVeniceCluster(numberOfController, numberOfServer, numberOfRouter); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(numberOfServer) + .numberOfRouters(numberOfRouter) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java index f0b0b4a133a..54eb22f5974 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerAfterDeletingSstFilesWithActiveActiveIngestion.java @@ -35,6 +35,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; @@ -123,18 +124,18 @@ public void setUp() throws Exception { serverProperties.put( CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME, "localhost:" + TestUtils.getFreePort()); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - NUMBER_OF_COLOS, - 1, - 1, - 2, - numServers, - 1, - NUMBER_OF_REPLICAS, - Optional.empty(), - Optional.empty(), - Optional.of(serverProperties), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_COLOS) + .numberOfClusters(1) + .numberOfParentControllers(1) + .numberOfChildControllers(2) + .numberOfServers(numServers) + .numberOfRouters(1) + .replicationFactor(NUMBER_OF_REPLICAS) + .forkServer(false) + .serverProperties(serverProperties); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); List childDatacenters = multiRegionMultiClusterWrapper.getChildRegions(); List parentControllers = multiRegionMultiClusterWrapper.getParentControllers(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java index 60d2eb5f31a..301c508e4c4 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/restart/TestRestartServerDuringIngestion.java @@ -14,6 +14,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.PubSubBrokerWrapper; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -74,8 +75,16 @@ public void setUp() { int numberOfRouter = 1; int replicaFactor = 1; int partitionSize = 1000; - cluster = ServiceFactory - .getVeniceCluster(numberOfController, 0, numberOfRouter, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(0) + .numberOfRouters(numberOfRouter) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); pubSubProducerAdapterFactory = cluster.getPubSubBrokerWrapper().getPubSubClientsFactory().getProducerAdapterFactory(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java index df162487768..6763cc66f93 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/ReplicaFailoverTest.java @@ -5,6 +5,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -52,7 +53,12 @@ public class ReplicaFailoverTest { @BeforeClass public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = ServiceFactory.getVeniceCluster(1, REPLICATION_FACTOR, 0, REPLICATION_FACTOR); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(REPLICATION_FACTOR) + .numberOfRouters(0) + .replicationFactor(REPLICATION_FACTOR) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); Properties props = new Properties(); props.setProperty(ConfigKeys.ROUTER_STATEFUL_HEALTHCHECK_ENABLED, "true"); props.setProperty(ConfigKeys.ROUTER_ASYNC_START_ENABLED, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java index aaa1da09a29..119ef28a86d 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestBlobDiscovery.java @@ -31,6 +31,7 @@ import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceControllerWrapper; import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper; +import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -46,7 +47,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -81,18 +81,18 @@ public void setUp() { Properties props = new Properties(); props.put(OFFLINE_JOB_START_TIMEOUT_MS, "180000"); - multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper( - 1, - 2, - 1, - 1, - 3, - 1, - 3, - Optional.of(props), - Optional.empty(), - Optional.empty(), - false); + VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder = + new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1) + .numberOfClusters(2) + .numberOfParentControllers(1) + .numberOfChildControllers(1) + .numberOfServers(3) + .numberOfRouters(1) + .replicationFactor(3) + .forkServer(false) + .parentControllerProperties(props); + multiRegionMultiClusterWrapper = + ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build()); multiClusterVenice = multiRegionMultiClusterWrapper.getChildRegions().get(0); String[] clusterNames = multiClusterVenice.getClusterNames(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java index 67ed54852be..aaf8e446be3 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestConnectionWarmingForApacheAsyncClient.java @@ -3,6 +3,7 @@ import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.client.exceptions.VeniceClientException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -26,7 +27,15 @@ public class TestConnectionWarmingForApacheAsyncClient { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 0, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); Properties routerProperties = new Properties(); routerProperties.put(ConfigKeys.ROUTER_STORAGE_NODE_CLIENT_TYPE, StorageNodeClientType.APACHE_HTTP_ASYNC_CLIENT); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java index 79968b4f759..c080531a6c8 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRead.java @@ -28,6 +28,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -158,7 +159,16 @@ public void setUp() throws VeniceClientException, ExecutionException, Interrupte extraProperties.put(ConfigKeys.ROUTER_PER_STORE_ROUTER_QUOTA_BUFFER, 0.0); extraProperties.put(ConfigKeys.PARTICIPANT_MESSAGE_STORE_ENABLED, false); - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 1, 2, 100, true, false, extraProperties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .extraProperties(extraProperties) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); routerAddr = veniceCluster.getRandomRouterSslURL(); Properties serverProperties = new Properties(); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java index 80d3bf9273b..929b030c501 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestRouterAsyncStart.java @@ -3,6 +3,7 @@ import com.linkedin.venice.ConfigKeys; import com.linkedin.venice.client.exceptions.VeniceClientException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.router.httpclient.StorageNodeClientType; @@ -23,7 +24,9 @@ public class TestRouterAsyncStart { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); } @AfterClass(alwaysRun = true) diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java index c793a65afc7..1d7bc992e9f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/router/TestStreaming.java @@ -29,6 +29,7 @@ import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.D2TestUtils; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; @@ -111,7 +112,15 @@ public void setUp() throws InterruptedException, ExecutionException, VeniceClien System.setProperty("io.netty.leakDetection.level", "paranoid"); Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 2, 0, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(2) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); Properties serverFeatureProperties = new Properties(); serverFeatureProperties.put(VeniceServerWrapper.SERVER_ENABLE_SSL, "true"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java index c5884bd25d8..af97bb1bf43 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/samza/VeniceSystemFactoryTest.java @@ -11,6 +11,7 @@ import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.schema.writecompute.WriteComputeSchemaConverter; @@ -43,7 +44,9 @@ public class VeniceSystemFactoryTest { @BeforeClass public void setUp() { - cluster = ServiceFactory.getVeniceCluster(); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); } @AfterClass diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java index 6fb1c6a615c..bf7dca1aa8f 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/server/VeniceServerTest.java @@ -79,7 +79,9 @@ public class VeniceServerTest { @Test public void testStartServerWithDefaultConfigForTests() throws NoSuchFieldException, IllegalAccessException { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { TestVeniceServer server = cluster.getVeniceServers().get(0).getVeniceServer(); Assert.assertTrue(server.isStarted()); @@ -91,7 +93,9 @@ public void testStartServerWithDefaultConfigForTests() throws NoSuchFieldExcepti @Test public void testStartServerWhenEnableAllowlistCheckingFailed() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Assert.assertThrows(VeniceException.class, () -> { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); @@ -104,7 +108,9 @@ public void testStartServerWhenEnableAllowlistCheckingFailed() { @Test public void testStartServerWhenEnableAllowlistCheckingSuccessful() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); @@ -115,7 +121,9 @@ public void testStartServerWhenEnableAllowlistCheckingSuccessful() { @Test public void testCheckBeforeJoinCluster() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { VeniceServerWrapper server = cluster.getVeniceServers().get(0); StorageEngineRepository repository = server.getVeniceServer().getStorageService().getStorageEngineRepository(); @@ -176,7 +184,9 @@ public void testCheckBeforeJoinCluster() { @Test public void testCheckBeforeJointClusterBeforeHelixInitializingCluster() throws Exception { Thread serverAddingThread = null; - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(0, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(0).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { serverAddingThread = new Thread(() -> { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(false)); @@ -200,7 +210,9 @@ public void testCheckBeforeJointClusterBeforeHelixInitializingCluster() throws E @Test public void testStartServerAndShutdownWithPartitionAssignmentVerification() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 0, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(0).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); @@ -244,7 +256,12 @@ public void testMetadataFetchRequest() throws ExecutionException, InterruptedExc int servers = 6; int replicationFactor = 2; - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, servers, 0, replicationFactor); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(servers) + .numberOfRouters(0) + .replicationFactor(replicationFactor) + .build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options); CloseableHttpAsyncClient client = HttpClientUtils.getMinimalHttpClient(1, 1, Optional.of(SslUtils.getVeniceLocalSslFactory()))) { @@ -351,7 +368,9 @@ public void testMetadataFetchRequest() throws ExecutionException, InterruptedExc @Test(dataProvider = "True-and-False", dataProviderClass = DataProviderUtils.class) public void testVeniceServerWithD2(boolean https) throws Exception { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { String storeName = cluster.createStore(1); Map clusterToServerD2 = cluster.getClusterToServerD2(); String d2ServiceName = clusterToServerD2.get(cluster.getClusterName()); @@ -397,7 +416,9 @@ public void testStartServerWithSystemSchemaInitialization() { @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) public void testDropStorePartitionAsynchronously() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { VeniceServerWrapper server = cluster.getVeniceServers().get(0); Assert.assertTrue(server.getVeniceServer().isStarted()); @@ -435,7 +456,9 @@ public void testDropStorePartitionAsynchronously() { @Test(timeOut = TOTAL_TIMEOUT_FOR_LONG_TEST_MS) public void testDropStorePartitionSynchronously() { - try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(1, 1, 0)) { + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(0).build(); + try (VeniceClusterWrapper cluster = ServiceFactory.getVeniceCluster(options)) { Properties featureProperties = new Properties(); featureProperties.setProperty(SERVER_ENABLE_SERVER_ALLOW_LIST, Boolean.toString(true)); featureProperties.setProperty(SERVER_IS_AUTO_JOIN, Boolean.toString(true)); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java index 58cb7aa09fb..a64044f3428 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/ReadComputeValidationTest.java @@ -18,6 +18,7 @@ import com.linkedin.venice.exceptions.ErrorType; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -94,7 +95,15 @@ public class ReadComputeValidationTest { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Add one more server with fast-avro enabled Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java index bf48f492a7c..ba498876267 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/StorageNodeComputeTest.java @@ -20,6 +20,7 @@ import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.meta.Version; import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory; @@ -97,7 +98,15 @@ enum AvroImpl { @BeforeClass(alwaysRun = true) public void setUp() throws InterruptedException, ExecutionException, VeniceClientException { - veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(0) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Add one more server with all the bells and whistles: fast-avro, parallel batch get Properties serverProperties = new Properties(); serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java index e36062a8477..5b3665f94b6 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/storagenode/TestEarlyTermination.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.Version; @@ -50,7 +51,15 @@ public class TestEarlyTermination { @BeforeClass(alwaysRun = true) public void setUp() throws VeniceClientException { Utils.thisIsLocalhost(); - veniceCluster = ServiceFactory.getVeniceCluster(1, 0, 1, 2, 100, true, false); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(0) + .numberOfRouters(1) + .replicationFactor(2) + .partitionSize(100) + .sslToStorageNodes(true) + .sslToKafka(false) + .build(); + veniceCluster = ServiceFactory.getVeniceCluster(options); // Create store first storeName = Utils.getUniqueString("test_early_termination"); diff --git a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java index 8c2ff7aa4c6..2f2107e65f0 100644 --- a/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java +++ b/internal/venice-test-common/src/integrationTest/java/com/linkedin/venice/throttle/TestRouterReadQuotaThrottler.java @@ -12,6 +12,7 @@ import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.helix.HelixReadOnlySchemaRepository; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceRouterWrapper; import com.linkedin.venice.meta.Store; @@ -50,7 +51,16 @@ public void setUp() throws Exception { Properties properties = new Properties(); properties.put(ConfigKeys.ROUTER_PER_STORE_ROUTER_QUOTA_BUFFER, 0.0); - cluster = ServiceFactory.getVeniceCluster(1, 1, numberOfRouter, 1, 10000, false, false, properties); + VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1) + .numberOfServers(1) + .numberOfRouters(numberOfRouter) + .replicationFactor(1) + .partitionSize(10000) + .sslToStorageNodes(false) + .sslToKafka(false) + .extraProperties(properties) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); VersionCreationResponse response = cluster.getNewStoreVersion(); Assert.assertFalse(response.isError()); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java index 82e8f00fbd0..e9458752cd5 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/DaVinciPartialKeyLookupBenchmark.java @@ -10,6 +10,7 @@ import com.linkedin.venice.client.store.ComputeGenericRecord; import com.linkedin.venice.client.store.predicate.Predicate; import com.linkedin.venice.client.store.streaming.StreamingCallback; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -76,7 +77,9 @@ public static void main(String[] args) throws Exception { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = getVeniceCluster(options); String storeName = buildVectorStore(cluster); client = getGenericAvroDaVinciClient( diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java index 84fb77dbfcc..fe2b70779aa 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/IngestionBenchmarkInSingleProcess.java @@ -5,6 +5,7 @@ import com.linkedin.davinci.client.DaVinciClient; import com.linkedin.venice.exceptions.VeniceException; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -61,7 +62,9 @@ public class IngestionBenchmarkInSingleProcess { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = ServiceFactory.getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = ServiceFactory.getVeniceCluster(options); if (valueType.equals("FLOAT_VECTOR")) { storeName = buildFloatVectorStore(cluster, Integer.parseInt(valueLength)); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java index 154e87a6b6b..a8dd1cedf53 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/MixedIngestionBenchmark.java @@ -6,6 +6,7 @@ import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; import com.linkedin.venice.controllerapi.VersionCreationResponse; import com.linkedin.venice.integration.utils.ServiceFactory; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.integration.utils.VeniceServerWrapper; import com.linkedin.venice.meta.PersistenceType; @@ -60,7 +61,16 @@ public void setUp() throws Exception { Utils.thisIsLocalhost(); int numberOfController = 1; - cluster = ServiceFactory.getVeniceCluster(numberOfController, 0, 0, replicaFactor, partitionSize, false, false); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(numberOfController) + .numberOfServers(0) + .numberOfRouters(0) + .replicationFactor(replicaFactor) + .partitionSize(partitionSize) + .sslToStorageNodes(false) + .sslToKafka(false) + .build(); + cluster = ServiceFactory.getVeniceCluster(options); cluster.addVeniceServer(new Properties(getVeniceServerProperties())); // JMH benchmark relies on System.exit to finish one round of benchmark run, otherwise it will hang there. TestUtils.restoreSystemExit(); diff --git a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java index 42afd7d86cd..03451480199 100644 --- a/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java +++ b/internal/venice-test-common/src/jmh/java/com/linkedin/venice/benchmark/VeniceClientBenchmark.java @@ -6,6 +6,7 @@ import com.linkedin.venice.client.store.ClientConfig; import com.linkedin.venice.client.store.ClientFactory; import com.linkedin.venice.controllerapi.UpdateStoreQueryParams; +import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions; import com.linkedin.venice.integration.utils.VeniceClusterWrapper; import com.linkedin.venice.utils.TestUtils; import com.linkedin.venice.utils.Utils; @@ -55,7 +56,9 @@ public class VeniceClientBenchmark { @Setup public void setUp() throws Exception { Utils.thisIsLocalhost(); - cluster = getVeniceCluster(1, 1, 1); + VeniceClusterCreateOptions options = + new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build(); + cluster = getVeniceCluster(options); String storeName = buildStore(cluster); cluster.useControllerClient(c -> c.updateStore(storeName, new UpdateStoreQueryParams().setReadQuotaInCU(10000))); client = ClientFactory.getAndStartGenericAvroClient(