Skip to content

Commit

Permalink
[test] Update Venice cluster initialization to replace deprecated APIs (
Browse files Browse the repository at this point in the history 
#1495)

Refactored cluster initialization in tests by replacing calls to deprecated APIs with  
overloads that had numerous method parameters, using `VeniceClusterCreateOptions`  
and `VeniceMultiRegionClusterCreateOptions` instead. This change simplifies test setup,  
improves readability, and reduces complexity by using builder-based initialization.  

Also added support for passing `DynamicAccessController` to Venice controllers in E2E  
test setups to enable testing of access control code paths.
  • Loading branch information
@sushantmane
sushantmane authored Feb 3, 2025
1 parent 57bc957 commit 9ab47e5
Show file tree
Hide file tree
Showing 100 changed files with 1,186 additions and 809 deletions.
12 changes: 11 additions & 1 deletion ...e-avro-compatibility-test/src/test/java/com/linkedin/venice/VeniceClusterInitializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.linkedin.venice.exceptions.VeniceException;
import com.linkedin.venice.helix.HelixReadOnlySchemaRepository;
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.meta.Version;
import com.linkedin.venice.pubsub.PubSubProducerAdapterFactory;
Expand Down Expand Up @@ -80,7 +81,16 @@ public VeniceClusterInitializer(String storeName, int routerPort) {
Properties clusterConfig = new Properties();
clusterConfig.put(ConfigKeys.SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L);
clusterConfig.put(ConfigKeys.ROUTER_ENABLE_SSL, false);
this.veniceCluster = ServiceFactory.getVeniceCluster(1, 1, 0, 2, 100, false, false, clusterConfig);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1)
.numberOfServers(1)
.numberOfRouters(0)
.replicationFactor(2)
.partitionSize(100)
.sslToStorageNodes(false)
.sslToKafka(false)
.extraProperties(clusterConfig)
.build();
this.veniceCluster = ServiceFactory.getVeniceCluster(options);
Properties serverProperties = new Properties();
serverProperties.put(ConfigKeys.SERVER_COMPUTE_FAST_AVRO_ENABLED, true);
this.veniceCluster.addVeniceServer(new Properties(), serverProperties);
Expand Down
5 changes: 4 additions & 1 deletion ...common/src/integrationTest/java/com/linkedin/venice/consumer/ConsumerIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import com.linkedin.venice.controllerapi.UpdateStoreQueryParams;
import com.linkedin.venice.exceptions.VeniceMessageException;
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.kafka.protocol.KafkaMessageEnvelope;
import com.linkedin.venice.partitioner.DefaultVenicePartitioner;
Expand Down Expand Up @@ -103,7 +104,9 @@ public abstract class ConsumerIntegrationTest {

@BeforeClass
public void sharedSetUp() {
cluster = ServiceFactory.getVeniceCluster();
VeniceClusterCreateOptions options =
new VeniceClusterCreateOptions.Builder().numberOfControllers(1).numberOfServers(1).numberOfRouters(1).build();
cluster = ServiceFactory.getVeniceCluster(options);
controllerClient =
ControllerClient.constructClusterControllerClient(cluster.getClusterName(), cluster.getAllControllersURLs());

Expand Down
25 changes: 13 additions & 12 deletions ...integrationTest/java/com/linkedin/venice/consumer/TestBootstrappingChangelogConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceControllerWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceRouterWrapper;
import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper;
import com.linkedin.venice.integration.utils.ZkServerWrapper;
Expand Down Expand Up @@ -93,18 +94,18 @@ public void setUp() {
serverProperties.put(
CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME,
"localhost:" + TestUtils.getFreePort());
multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(
1,
1,
1,
1,
1,
1,
1,
Optional.empty(),
Optional.empty(),
Optional.of(serverProperties),
false);
VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder =
new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1)
.numberOfClusters(1)
.numberOfParentControllers(1)
.numberOfChildControllers(1)
.numberOfServers(1)
.numberOfRouters(1)
.replicationFactor(1)
.forkServer(false)
.serverProperties(serverProperties);
multiRegionMultiClusterWrapper =
ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build());

childDatacenters = multiRegionMultiClusterWrapper.getChildRegions();
parentControllers = multiRegionMultiClusterWrapper.getParentControllers();
Expand Down
25 changes: 13 additions & 12 deletions ...t-common/src/integrationTest/java/com/linkedin/venice/consumer/TestChangelogConsumer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceControllerWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceRouterWrapper;
import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper;
import com.linkedin.venice.integration.utils.ZkServerWrapper;
Expand Down Expand Up @@ -137,18 +138,18 @@ public void setUp() {
CHILD_DATA_CENTER_KAFKA_URL_PREFIX + "." + DEFAULT_PARENT_DATA_CENTER_REGION_NAME,
"localhost:" + TestUtils.getFreePort());
serverProperties.put(SERVER_AA_WC_WORKLOAD_PARALLEL_PROCESSING_ENABLED, isAAWCParallelProcessingEnabled());
multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(
1,
1,
1,
1,
1,
1,
1,
Optional.empty(),
Optional.empty(),
Optional.of(serverProperties),
false);
VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder =
new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(1)
.numberOfClusters(1)
.numberOfParentControllers(1)
.numberOfChildControllers(1)
.numberOfServers(1)
.numberOfRouters(1)
.replicationFactor(1)
.forkServer(false)
.serverProperties(serverProperties);
multiRegionMultiClusterWrapper =
ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build());

childDatacenters = multiRegionMultiClusterWrapper.getChildRegions();
parentControllers = multiRegionMultiClusterWrapper.getParentControllers();
Expand Down
26 changes: 13 additions & 13 deletions ...test-common/src/integrationTest/java/com/linkedin/venice/controller/AdminToolE2ETest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceControllerWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper;
import com.linkedin.venice.meta.StoreInfo;
import com.linkedin.venice.utils.TestUtils;
Expand All @@ -23,7 +24,6 @@
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -51,18 +51,18 @@ public void setUp() {
parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_META_SYSTEM_STORE, "false");
parentControllerProperties.setProperty(CONTROLLER_AUTO_MATERIALIZE_DAVINCI_PUSH_STATUS_SYSTEM_STORE, "false");

multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(
NUMBER_OF_CHILD_DATACENTERS,
NUMBER_OF_CLUSTERS,
2,
2,
2,
2,
1,
Optional.of(parentControllerProperties),
Optional.empty(),
Optional.empty(),
false);
VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder =
new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS)
.numberOfClusters(NUMBER_OF_CLUSTERS)
.numberOfParentControllers(2)
.numberOfChildControllers(2)
.numberOfServers(2)
.numberOfRouters(2)
.replicationFactor(1)
.forkServer(false)
.parentControllerProperties(parentControllerProperties);
multiRegionMultiClusterWrapper =
ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build());
childDatacenters = multiRegionMultiClusterWrapper.getChildRegions();
clusterNames = multiRegionMultiClusterWrapper.getClusterNames();
}
Expand Down
12 changes: 11 additions & 1 deletion ...common/src/integrationTest/java/com/linkedin/venice/controller/TestAdminToolEndToEnd.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import com.linkedin.venice.helix.HelixReadOnlyLiveClusterConfigRepository;
import com.linkedin.venice.helix.ZkClientFactory;
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceServerWrapper;
import com.linkedin.venice.meta.Version;
Expand Down Expand Up @@ -47,7 +48,16 @@ public void setUp() {
properties.setProperty(LOCAL_REGION_NAME, "dc-0");
properties.setProperty(ALLOW_CLUSTER_WIPE, "true");
properties.setProperty(TOPIC_CLEANUP_DELAY_FACTOR, "0");
venice = ServiceFactory.getVeniceCluster(1, 1, 1, 1, 100000, false, false, properties);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(1)
.numberOfServers(1)
.numberOfRouters(1)
.replicationFactor(1)
.partitionSize(100000)
.sslToStorageNodes(false)
.sslToKafka(false)
.extraProperties(properties)
.build();
venice = ServiceFactory.getVeniceCluster(options);
clusterName = venice.getClusterName();
}

Expand Down
26 changes: 14 additions & 12 deletions ...st-common/src/integrationTest/java/com/linkedin/venice/controller/TestFabricBuildout.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceControllerWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceMultiRegionClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceTwoLayerMultiRegionMultiClusterWrapper;
import com.linkedin.venice.meta.StoreInfo;
import com.linkedin.venice.utils.TestUtils;
Expand Down Expand Up @@ -56,18 +57,19 @@ public void setUp() {
childControllerProperties.setProperty(ALLOW_CLUSTER_WIPE, "true");
Properties serverProperties = new Properties();
serverProperties.put(SERVER_PROMOTION_TO_LEADER_REPLICA_DELAY_SECONDS, 1L);
multiRegionMultiClusterWrapper = ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(
NUMBER_OF_CHILD_DATACENTERS,
NUMBER_OF_CLUSTERS,
1,
1,
1,
1,
1,
Optional.empty(),
Optional.of(childControllerProperties),
Optional.of(serverProperties),
false);
VeniceMultiRegionClusterCreateOptions.Builder optionsBuilder =
new VeniceMultiRegionClusterCreateOptions.Builder().numberOfRegions(NUMBER_OF_CHILD_DATACENTERS)
.numberOfClusters(NUMBER_OF_CLUSTERS)
.numberOfParentControllers(1)
.numberOfChildControllers(1)
.numberOfServers(1)
.numberOfRouters(1)
.replicationFactor(1)
.forkServer(false)
.childControllerProperties(childControllerProperties)
.serverProperties(serverProperties);
multiRegionMultiClusterWrapper =
ServiceFactory.getVeniceTwoLayerMultiRegionMultiClusterWrapper(optionsBuilder.build());

childDatacenters = multiRegionMultiClusterWrapper.getChildRegions();
parentControllers = multiRegionMultiClusterWrapper.getParentControllers();
Expand Down
50 changes: 43 additions & 7 deletions ...st-common/src/integrationTest/java/com/linkedin/venice/controller/TestHAASController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.linkedin.venice.controllerapi.NewStoreResponse;
import com.linkedin.venice.integration.utils.HelixAsAServiceWrapper;
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceControllerWrapper;
import com.linkedin.venice.integration.utils.ZkServerWrapper;
Expand Down Expand Up @@ -68,7 +69,12 @@ public void setUp() {

@Test(timeOut = 60 * Time.MS_PER_SECOND)
public void testClusterResourceInstanceTag() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0)
.numberOfServers(0)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
String instanceTag = "GENERAL";
String controllerClusterName = "venice-controllers";
Expand All @@ -89,7 +95,12 @@ public void testClusterResourceInstanceTag() {

@Test(timeOut = 60 * Time.MS_PER_SECOND)
public void testClusterResourceEmptyInstanceTag() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0)
.numberOfServers(0)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
String instanceTag = "";
String controllerClusterName = "venice-controllers";
Expand All @@ -110,7 +121,12 @@ public void testClusterResourceEmptyInstanceTag() {

@Test(timeOut = 60 * Time.MS_PER_SECOND)
public void testStartHAASHelixControllerAsControllerClusterLeader() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0)
.numberOfServers(0)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
VeniceControllerWrapper controllerWrapper = venice.addVeniceController(enableControllerClusterHAASProperties);
waitForNonDeterministicAssertion(
Expand Down Expand Up @@ -141,7 +157,12 @@ public void testStartHAASHelixControllerAsControllerClusterLeader() {

@Test(timeOut = 120 * Time.MS_PER_SECOND)
public void testTransitionToHAASControllerAsControllerClusterLeader() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3)
.numberOfServers(1)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store"));
venice.useControllerClient(
Expand Down Expand Up @@ -182,7 +203,12 @@ public void testTransitionToHAASControllerAsControllerClusterLeader() {

@Test(timeOut = 90 * Time.MS_PER_SECOND)
public void testStartHAASControllerAsStorageClusterLeader() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0)
.numberOfServers(0)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
VeniceControllerWrapper controllerWrapper =
venice.addVeniceController(enableControllerAndStorageClusterHAASProperties);
Expand Down Expand Up @@ -212,7 +238,12 @@ public void testStartHAASControllerAsStorageClusterLeader() {

@Test(timeOut = 180 * Time.MS_PER_SECOND)
public void testTransitionToHAASControllerAsStorageClusterLeader() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(3, 1, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(3)
.numberOfServers(1)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {

NewStoreResponse response = venice.getNewStore(Utils.getUniqueString("venice-store"));
Expand Down Expand Up @@ -362,7 +393,12 @@ public void testCloudConfig() {

@Test(timeOut = 90 * Time.MS_PER_SECOND)
public void testHelixUnknownInstanceOperation() {
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(0, 0, 0, 1);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(0)
.numberOfServers(0)
.numberOfRouters(0)
.replicationFactor(1)
.build();
try (VeniceClusterWrapper venice = ServiceFactory.getVeniceCluster(options);
HelixAsAServiceWrapper helixAsAServiceWrapper = startAndWaitForHAASToBeAvailable(venice.getZk().getAddress())) {
VeniceControllerWrapper controllerWrapper =
venice.addVeniceController(enableControllerAndStorageClusterHAASProperties);
Expand Down
11 changes: 10 additions & 1 deletion ...src/integrationTest/java/com/linkedin/venice/controller/TestHolisticSeverHealthCheck.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import com.linkedin.venice.helix.HelixCustomizedViewOfflinePushRepository;
import com.linkedin.venice.helix.ResourceAssignment;
import com.linkedin.venice.integration.utils.ServiceFactory;
import com.linkedin.venice.integration.utils.VeniceClusterCreateOptions;
import com.linkedin.venice.integration.utils.VeniceClusterWrapper;
import com.linkedin.venice.integration.utils.VeniceServerWrapper;
import com.linkedin.venice.meta.PartitionAssignment;
Expand Down Expand Up @@ -42,7 +43,15 @@ public class TestHolisticSeverHealthCheck {
@BeforeClass
public void setUp() {
int numOfController = 1;
cluster = ServiceFactory.getVeniceCluster(numOfController, 2, 1, replicaFactor, partitionSize, false, false);
VeniceClusterCreateOptions options = new VeniceClusterCreateOptions.Builder().numberOfControllers(numOfController)
.numberOfServers(2)
.numberOfRouters(1)
.replicationFactor(replicaFactor)
.partitionSize(partitionSize)
.sslToStorageNodes(false)
.sslToKafka(false)
.build();
cluster = ServiceFactory.getVeniceCluster(options);

Properties routerProperties = new Properties();
cluster.addVeniceRouter(routerProperties);
Expand Down
Loading

0 comments on commit 9ab47e5

Please sign in to comment.