[DRAFT] Experimental minimal S3 plugin

docker/s3/.gitignore → docker/.gitignore

@@ -2,7 +2,4 @@
 .env
 
 # Demo keys
-*.pem
-
-# Backend volumes
-minio/
+*.pem

docker/Makefile

@@ -0,0 +1,26 @@
+all:
+
+# Generate keys for S3 plugin
+rsa-keys:
+	openssl genrsa -out private.pem 512
+	openssl rsa -in private.pem -outform PEM -out public.pem -pubout
+<<<<<<<< HEAD:docker/Makefile
+
+# Topic Defaults
+t=t1
+p=6
+segment=100000000 # 100M
+retention=1000000000 # 1G
+local_retention=300000000 # 300M
+
+kafka-topic:
+	${KAFKA_HOME}/bin/kafka-topics.sh \
+		--bootstrap-server localhost:9092 \
+		--create \
+		--config segment.bytes=$(segment) \
+		--config retention.bytes=$(retention) \
+		--config local.retention.bytes=$(local_retention) \
+		--partitions $(p) \
+		--topic $(t)
+========
+>>>>>>>> ff77535 (feat: add local tiered storage impl to docker options):docker/s3/Makefile

docker/README.md

@@ -11,18 +11,51 @@ Docker compose environment to try out tiered storage plugins.
 
 ## How to use
 
-### S3
-
-> on [./s3](./s3) directory
+### Pre-requisites
 
 Build plugin before starting Docker compose:
 
+#### Build plugin libraries
+
 On root directory:
 
 ```shell
-./gradlew clean s3:installDist
+./gradlew clean installDist
+```
+
+#### Using Encryption mechanisms
+
+Generate RSA key pair:
+
+```shell
+make rsa-keys
+```
+
+and set paths on `compose.yml` file:
+
+```yaml
+  kafka:
+    # ...
+    volumes:
+      # ...
+      - ./private.pem:/kafka/plugins/private.pem
+      - ./public.pem:/kafka/plugins/public.pem
+    command:
+      - kafka-server-start.sh
+      - config/server.properties
+      # ...
+      - --override
+      - rsm.config.s3.public_key_pem=/kafka/plugins/public.pem
+      - --override
+      - rsm.config.s3.private_key_pem=/kafka/plugins/private.pem
+      # ...
 ```
 
+### S3
+
+> for AWS S3 try [./s3](./s3) directory and
+> for Minio S3 try [./s3-minio](./s3-minio) directory
+
 `compose.yml` is mounting the distribution directory:
 
 ```yaml
@@ -66,41 +99,13 @@ and use variables on `compose.yml`:
       - rsm.config.s3.client.aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
 ```
 
-#### Using Encryption mechanisms
-
-Generate RSA key pair:
-
-```shell
-make rsa-keys
-```
-
-and set paths on `compose.yml` file:
-
-```yaml
-  kafka:
-    # ...
-    volumes:
-      # ...
-      - ./private.pem:/kafka/plugins/private.pem
-      - ./public.pem:/kafka/plugins/public.pem
-    command:
-      - kafka-server-start.sh
-      - config/server.properties
-      # ...
-      - --override
-      - rsm.config.s3.public_key_pem=/kafka/plugins/public.pem
-      - --override
-      - rsm.config.s3.private_key_pem=/kafka/plugins/private.pem
-      # ...
-```
-
 
 ### Kafka
 
 Creating topics with Tiered storage:
 
 ```shell
-make topic
+make kafka-topic
 ```
 
 creates a topic t1 with 6 partitions, 10MB segments, retention bytes 100MB, and 20MB local retention.

docker/kafka/Dockerfile

@@ -1,6 +1,6 @@
 FROM fedora:37 AS base
 
-ENV JAVA_VERSION 17
+ENV JAVA_VERSION 11
 RUN dnf install -y java-${JAVA_VERSION}-openjdk-devel git
 
 ENV JAVA_HOME /usr/lib/jvm/java-${JAVA_VERSION}-openjdk
@@ -12,11 +12,12 @@ RUN git clone ${AK_REPO}
 WORKDIR /kafka
 RUN git checkout ${AK_BRANCH}
 
-RUN ./gradlew releaseTarGz
+ENV SCALA_VERSION 2.12
+RUN ./gradlew -PscalaVersion=${SCALA_VERSION} releaseTarGz
 
 FROM eclipse-temurin:17-jre AS kafka
 
-ENV SCALA_VERSION 2.13
+ENV SCALA_VERSION 2.12
 ARG KAFKA_VERSION=3.4.1-SNAPSHOT
 
 COPY --from=base /kafka/core/build/distributions/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz ./kafka.tgz

docker/s3-minio/compose.yml

@@ -0,0 +1,70 @@
+version: '3.8'
+services:
+  zookeeper:
+    image: kafka-ts-s3-minio
+    command:
+      - zookeeper-server-start.sh
+      - config/zookeeper.properties
+
+  kafka:
+    image: kafka-ts-s3-minio
+    build:
+      context: ../kafka/.
+      args:
+        - AK_REPO=https://github.com/aiven/kafka
+        - AK_BRANCH=3.0-2022-03-31-tiered-storage
+        - KAFKA_VERSION=3.0.1-SNAPSHOT
+    ports:
+      - "9092:9092"
+    volumes:
+      # mount plugin
+      - ./../../s3-minimal/build/install/s3-minimal:/kafka/plugins/tiered-storage-s3
+    command:
+      - kafka-server-start.sh
+      - config/server.properties
+      - --override
+      - zookeeper.connect=zookeeper:2181
+      - --override
+      - listeners=BROKER://kafka:29092,EXTERNAL://kafka:9092
+      - --override
+      - advertised.listeners=BROKER://kafka:29092,EXTERNAL://localhost:9092
+      - --override
+      - listener.security.protocol.map=BROKER:PLAINTEXT,EXTERNAL:PLAINTEXT
+      - --override
+      - inter.broker.listener.name=BROKER
+      # Enable Tiered storage
+      - --override
+      - remote.log.storage.system.enable=true
+      # Tiered Storage: RLMM config
+      - --override
+      - remote.log.metadata.manager.class.name=org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManager
+      - --override
+      - rlmm.config.remote.log.metadata.common.client.bootstrap.servers=kafka:29092
+      - --override
+      - rlmm.config.remote.log.metadata.topic.replication.factor=1
+      # Tiered storage S3 plugin
+      - --override
+      - remote.log.storage.manager.class.name=io.aiven.kafka.tieredstorage.s3.S3TieredStorage
+      - --override
+      - remote.log.storage.manager.class.path=/kafka/plugins/tiered-storage-s3/*
+      - --override
+      - rsm.config.s3.bucket.name=kafka-ts-v1
+      - --override
+      - rsm.config.s3.endpoint.url=http://minio:9000
+      - --override
+      - rsm.config.s3.aws.credentials.provider.class=com.amazonaws.auth.AWSStaticCredentialsProvider
+      - --override
+      - rsm.config.s3.aws.access.key.id=admin
+      - --override
+      - rsm.config.s3.aws.secret.access.key=password
+
+  minio:
+    image: quay.io/minio/minio
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DOMAIN=minio
+    ports:
+      - "9000:9000"
+      - "9090:9090"
+    command: server /data --console-address ":9090"

docker/s3/compose.yml

@@ -1,33 +1,24 @@
 version: '3.8'
 services:
   zookeeper:
-    image: kafka
-    build:
-      context: ../kafka/.
-      args:
-        - AK_REPO=https://github.com/aiven/kafka
-        - AK_BRANCH=3.3-2022-10-06-tiered-storage
-        - KAFKA_VERSION=3.3.2-SNAPSHOT
+    image: kafka-ts-s3-aws
     command:
       - zookeeper-server-start.sh
       - config/zookeeper.properties
 
   kafka:
-    image: kafka
+    image: kafka-ts-s3-aws
     build:
       context: ../kafka/.
       args:
         - AK_REPO=https://github.com/aiven/kafka
-        - AK_BRANCH=3.3-2022-10-06-tiered-storage
-        - KAFKA_VERSION=3.3.2-SNAPSHOT
+        - AK_BRANCH=3.0-2022-03-31-tiered-storage
+        - KAFKA_VERSION=3.0.1-SNAPSHOT
     ports:
       - "9092:9092"
     volumes:
       # mount plugin
-      - ./../../s3/build/install/s3:/kafka/plugins/tiered-storage-s3
-      # encryption keys
-      - ./private.pem:/kafka/plugins/private.pem
-      - ./public.pem:/kafka/plugins/public.pem
+      - ./../../s3-minimal/build/install/s3-minimal:/kafka/plugins/tiered-storage-s3
     command:
       - kafka-server-start.sh
       - config/server.properties
@@ -44,51 +35,27 @@ services:
       # Enable Tiered storage
       - --override
       - remote.log.storage.system.enable=true
+      # Tiered Storage: RLMM config
       - --override
-      - remote.log.storage.manager.class.name=io.aiven.kafka.tiered.storage.s3.S3RemoteStorageManager
+      - remote.log.metadata.manager.class.name=org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManager
       - --override
-      - remote.log.storage.manager.class.path=/kafka/plugins/tiered-storage-s3/*
+      - rlmm.config.remote.log.metadata.common.client.bootstrap.servers=kafka:29092
       - --override
-      - remote.log.metadata.manager.class.name=org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManager
+      - rlmm.config.remote.log.metadata.topic.replication.factor=1
       # Tiered storage S3 plugin
       - --override
-      - rsm.config.s3.bucket.name=kafka-ts-us-east-1
+      - remote.log.storage.manager.class.name=io.aiven.kafka.tieredstorage.s3.S3TieredStorage
       - --override
-      - rsm.config.s3.region=us-east-1
-      #- --override
-      #- rsm.config.s3.endpoint.url=http://minio:9090
+      - remote.log.storage.manager.class.path=/kafka/plugins/tiered-storage-s3/*
       - --override
-      - rsm.config.s3.credentials.provider.class=com.amazonaws.auth.AWSStaticCredentialsProvider
+      - rsm.config.s3.bucket.name=aiven-jeqo-test1
       - --override
-      - rsm.config.s3.client.aws_access_key_id=${AWS_ACCESS_KEY_ID}
+      - rsm.config.s3.object.prefix=kafka-ts-v1/cluster1
       - --override
-      - rsm.config.s3.client.aws_secret_access_key=${AWS_SECRET_ACCESS_KEY}
+      - rsm.config.s3.aws.region=eu-north-1
       - --override
-      - rsm.config.s3.public_key_pem=/kafka/plugins/public.pem
+      - rsm.config.s3.aws.credentials.provider.class=com.amazonaws.auth.AWSStaticCredentialsProvider
       - --override
-      - rsm.config.s3.private_key_pem=/kafka/plugins/private.pem
-      # Tiered Storage: RLMM config
+      - rsm.config.s3.aws.access.key.id=${AWS_ACCESS_KEY_ID}
       - --override
-      - rlmm.config.remote.log.metadata.common.client.bootstrap.servers=kafka:29092
-      - --override
-      - rlmm.config.remote.log.metadata.topic.replication.factor=1
-
-  # Experimental services
-  localstack:
-    image: localstack/localstack:2.0.0
-    environment:
-      - SERVICES=s3
-    ports:
-      - "4566:4566"
-
-  minio:
-    image: quay.io/minio/minio
-    environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-    ports:
-      - "9000:9000"
-      - "9090:9090"
-    volumes:
-      - ./minio/data:/data
-    command: server /data --console-address ":9090"
+      - rsm.config.s3.aws.secret.access.key=${AWS_SECRET_ACCESS_KEY}