3.10.0 acl disabled override

.github/workflows/pr-build-and-test.yml

@@ -37,7 +37,7 @@ jobs:
           mkdir test-results
           find . -type d -name "*surefire*" -exec cp --parents -R {} test-results/ \;
           zip -r test-results.zip test-results
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         name: upload test-results
         if: failure()
         with:

.gitignore

@@ -8,9 +8,12 @@ target
 *.svg
 core/*.json
 *.json
+!**/templates/*.json
 *.retry
 *.pem
 *.hcl
+**/inventory.txt
+**/tf_ansible_vars_file.yml
 **/.terraform
 **/terraform.tfstate
 **/terraform.tfstate.backup

.sdkmanrc

@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=17.0.14-tem

benchmark-framework/pom.xml

@@ -51,6 +51,11 @@
             <artifactId>driver-bookkeeper</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>driver-gateway</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>driver-jms</artifactId>

benchmark-framework/src/main/resources/log4j2.yaml

@@ -33,7 +33,12 @@ Configuration:
           size: 100MB
       DefaultRollOverStrategy:
         max: 10
-  Loggers: 
+  Loggers:
+    logger:
+      - name: io.openmessaging.benchmark
+        level: info
+      - name: io.openmessaging.benchmark.driver.gateway
+        level: debug
     Root:
       level: info
       additivity: false

benchmark-framework/src/test/java/io/openmessaging/benchmark/utils/PaddingDecimalFormatTest.java

@@ -15,11 +15,13 @@
 
 import static org.assertj.core.api.Assertions.assertThat;
 
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 
 class PaddingDecimalFormatTest {
 
     @Test
+    @Disabled("This test seem to be platform dependent")
     void format() {
         PaddingDecimalFormat format = new PaddingDecimalFormat("0.0", 7);
         assertThat(format.format(1L)).isEqualTo("    1.0");

docker-compose.yml

@@ -0,0 +1,108 @@
+services:
+  zookeeper:
+    image: "confluentinc/cp-zookeeper:latest"
+    container_name: openmessaging-benchmark-zookeeper
+    restart: always
+    ports:
+      - "22181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+    healthcheck:
+      test: echo srvr | nc zookeeper 2181 || exit 1
+      retries: 20
+      interval: 10s
+  kafka-1:
+    image: "confluentinc/cp-kafka:latest"
+    container_name: openmessaging-benchmark-kafka-1
+    restart: always
+    ports:
+      - "19092:19092"
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_LISTENERS: "INTERNAL://kafka-1:9092,EXTERNAL://:19092"
+      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-1:9092,EXTERNAL://localhost:19092"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_BROKER_ID: 1
+    healthcheck:
+      test: nc -zv kafka-1 9092 || exit 1
+      interval: 10s
+      retries: 25
+      start_period: 20s
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+  kafka-2:
+    image: confluentinc/cp-kafka
+    container_name: openmessaging-benchmark-kafka-2
+    hostname: kafka-2
+    ports:
+      - 19093:19093
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_LISTENERS: "INTERNAL://kafka-2:9093,EXTERNAL://:19093"
+      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-2:9093,EXTERNAL://localhost:19093"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_BROKER_ID: 2
+    healthcheck:
+      test: nc -zv kafka-2 9093 || exit 1
+      interval: 10s
+      retries: 25
+      start_period: 20s
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+  kafka-3:
+    image: confluentinc/cp-kafka
+    container_name: openmessaging-benchmark-kafka-3
+    hostname: kafka-3
+    ports:
+      - 19094:19094
+    environment:
+      KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
+      KAFKA_LISTENERS: "INTERNAL://kafka-3:9094,EXTERNAL://:19094"
+      KAFKA_ADVERTISED_LISTENERS: "INTERNAL://kafka-3:9094,EXTERNAL://localhost:19094"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT"
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_BROKER_ID: 3
+    healthcheck:
+      test: nc -zv kafka-3 9094 || exit 1
+      interval: 10s
+      retries: 25
+      start_period: 20s
+    depends_on:
+      zookeeper:
+        condition: service_healthy
+
+  conduktor-gateway:
+    image: harbor.cdkt.dev/conduktor/conduktor-gateway:3.8.0-rc1
+    hostname: conduktor-gateway
+    container_name: openmessaging-benchmark-conduktor-gateway
+    environment:
+      KAFKA_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9093,kafka-3:9094
+      JAVA_OPTS: "-XX:UseSVE=0"
+      VAULT_TOKEN: vault-plaintext-root-token
+    ports:
+      - "8888:8888"
+      - "6969:6969"
+    healthcheck:
+      test: curl localhost:8888/health
+      interval: 5s
+      retries: 25
+    depends_on:
+      kafka-1:
+        condition: service_healthy
+      kafka-2:
+        condition: service_healthy
+      kafka-3:
+        condition: service_healthy
+  kafka-client:
+      image: confluentinc/cp-kafka:latest
+      hostname: kafka-client
+      container_name: openmessaging-benchmark-kafka-client
+      command: sleep infinity
+      volumes:
+      - ${PWD}:${PWD}
+      - ~/.m2:$HOME/.m2
+      working_dir: ${PWD}

driver-gateway/README.md

@@ -0,0 +1,29 @@
+# Conduktor Gateway benchmarks
+
+NOTE: This is a slightly modified version of Apache Kafka Benchmarks for Conduktor Gateway
+
+## How to run the benchmarks
+
+Pre-requisites:
+- terraform 1.8+ (older version may also work)
+- ansible 2.16 (older version may also work)
+- AWS CLI setup with account with permission on EC2, VPC and API Gateway
+
+1. Create an SSH key named `kafka-aws` in `~/.ssh/` with `ssh-keygen -f ~/.ssh/kafka_aws` **without passphrase**
+2. Build the`openmessaging-benchmark` project with `mvn clean install`
+3. Go to the `driver-gateway/deploy/ssd-deployment` module
+4. Initialize terraform with `terraform init`
+5. Create the infrastructure with `terraform apply`
+   1. You can change the value of num_instances using -var='num_instances={"client"=5, "gateway"=4, "kafka"=2}'
+   2. You might also need to change the AWS profile with -var='aws_profile=your_profile'
+   3. With SSO the session does expire so you might need to run `aws sso login --profile your_profile` before running terraform
+6. Export your harbor creds https://harbor.cdkt.dev/
+   1. Click your name in the top right corner and select User Profile then take your username and CLI secret
+   2. export REGISTRY_USERNAME=<registry login>
+   3. export REGISTRY_PASSWORD=<registry api token>
+7. Setup nodes with `ansible-playbook --user ec2-user --inventory-file inventory.ini deploy.yaml`
+8. Connect to one benchmark worker node with `ssh -i ~/.ssh/kafka_aws ec2-user@$(terraform output client_ssh_host | tr -d '"')`
+9. Go to benchmark directory with `cd /opt/benchmark`
+10. Run the benchmark with `sudo bin/benchmark --drivers driver-gateway/gateway-latency.yaml workloads/100-topic-4-partitions-1kb-4p-4c-500k.yaml;`
+11. Download reports from nodes with `scp -i ~/.ssh/kafka_aws ec2-user@$(terraform output client_ssh_host | tr -d '"'):/opt/benchmark/*.json ./reports`
+

driver-gateway/deploy/README.md

@@ -0,0 +1,37 @@
+# Gateway Deployments
+
+There are two types of deployment found under ssd-deployment and hdd-deployment folders:
+
+- SSD with the i3 instance type that has fast NVMe drives as local instance stores
+- HDD with the d2 instance type that has fast HDD sequential IO drives as local instance stores
+
+Customize the instance types in the terraform.tfvars.
+
+If you choose larger instances, they come with more drives. To include those in the benchmarks you must:
+
+- update the Ansible script to include them in the mount and filesystem tasks
+- update the server.properties to include them in the logs.dir config
+
+NOTE: When using d2 instances, the instance stores are not automatically generated. You must add them to the provision-kafka-aws.tf file.
+
+For instructions on how to run a benchmark see the [Kafka instructions](http://openmessaging.cloud/docs/benchmarks/kafka/). Run the Terraform and Ansible commands from either the ssd-deployment or hdd-deployment folder.
+
+## AWS Instance Types
+
+| Instance | vCPU | RAM (GB) | Instance Store Drives | Network Baseline | Network Burst |
+| -- | -- | -- | -- | -- | -- |
+| i3.large | 2 | 15.25 | 1 x 475 NVMe SSD |	0.74 | 10 |
+| i3.xlarge | 4 | 30.5 | 1 x 950 NVMe SSD |	1.24 | 10 |
+| i3.2xlarge | 8 | 61 | 1 x 1,900 NVMe SSD | 2.48 | 10 |
+| i3.4large | 16 | 122 | 2 x 1,900 NVMe SSD | 4.96 | 10 |
+| i3.8xlarge | 32 | 244 | 4 x 1,900 NVMe SSD | 10 | 10 |
+| d2.xlarge | 4 | 30.5 | 3 x 2 TB HDD |	1.24 | 1.24 |
+| d2.2xlarge | 8 | 61 | 6 x 2 TB HDD |	2.48 | 2.48 |
+| d2.4large | 16 | 122 | 12 x 2 TB HDD | 4.96 | 4.96 |
+| d2.8xlarge | 36 | 244 | 24 x 2 TB HDD | 9.85 | 9.85 |
+
+## Compression
+
+When using 4 client VMs or less you may see lower throughput when using compression. Compression is performed by the producers and consumers only (when using defaults) and clients need to be spread across more VMs to see any throughput gains.
+
+Obviously, throughput may not be your primary goal when using compression.

driver-gateway/deploy/ssd-deployment/.tool-versions

@@ -0,0 +1,2 @@
+terraform 1.8.3
+ansible 2.16.6

driver-gateway/deploy/ssd-deployment/ansible.cfg

@@ -0,0 +1,18 @@
+[defaults]
+interpreter_python = /usr/bin/python3
+private_key_file=~/.ssh/kafka_aws
+remote_user=ec2-user
+# ansible optimization https://www.redhat.com/sysadmin/faster-ansible-playbook-execution
+forks=15
+callbacks_enabled = timer, profile_tasks, profile_roles
+pipelining=true
+host_key_checking=false
+
+[privilege_escalation]
+become=true
+become_method=sudo
+become_user=root
+
+[ssh_connection]
+ssh_args=-o ServerAliveInterval=60 -o ControlMaster=auto -o ControlPersist=60s
+retries=10