AWS CloudWatch Event Sink Implementation

gradle/libs.versions.toml

@@ -72,6 +72,7 @@ jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api",
 jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" }
 javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
 junit-bom = { module = "org.junit:junit-bom", version = "5.13.4" }
+localstack = { module = "org.testcontainers:localstack", version = "1.19.7" }
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.18" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.15.3" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }

runtime/service/build.gradle.kts

@@ -101,6 +101,7 @@ dependencies {
   implementation("software.amazon.awssdk:sts")
   implementation("software.amazon.awssdk:iam-policy-builder")
   implementation("software.amazon.awssdk:s3")
+  implementation("software.amazon.awssdk:cloudwatchlogs")
   implementation("software.amazon.awssdk:apache-client") {
     exclude("commons-logging", "commons-logging")
   }
@@ -142,6 +143,9 @@ dependencies {
   testImplementation("io.quarkus:quarkus-rest-client")
   testImplementation("io.quarkus:quarkus-rest-client-jackson")
   testImplementation("io.rest-assured:rest-assured")
+  testImplementation(libs.localstack)
+  testImplementation("org.testcontainers:testcontainers")
+  testImplementation(project(":polaris-container-spec-helper"))
 
   testImplementation(libs.threeten.extra)
   testImplementation(libs.hawkular.agent.prometheus.scraper)

runtime/service/src/main/java/org/apache/polaris/service/events/PolarisEventListenerConfiguration.java

@@ -25,8 +25,8 @@
 @ConfigMapping(prefix = "polaris.event-listener")
 public interface PolarisEventListenerConfiguration {
   /**
-   * The type of the event listener to use. Must be a registered {@link
-   * org.apache.polaris.service.events.PolarisEventListener} identifier.
+   * The type of the event listener to use. Must be a registered {@link PolarisEventListener}
+   * identifier.
    */
   String type();
 }

runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/JsonEventListener.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.events.jsonEventListener;
+
+import java.util.HashMap;
+import org.apache.polaris.service.events.AfterTableRefreshedEvent;
+import org.apache.polaris.service.events.PolarisEventListener;
+
+/**
+ * Abstract base class from which all event sinks that output events in JSON format can extend.
+ *
+ * <p>This class provides a common framework for transforming Polaris events into JSON format and
+ * sending them to various destinations. Concrete implementations should override the {@link
+ * #transformAndSendEvent(HashMap)} method to define how the JSON event data should be transmitted
+ * or stored.
+ */
+public abstract class JsonEventListener extends PolarisEventListener {
+  protected abstract void transformAndSendEvent(HashMap<String, Object> properties);
+
+  @Override
+  public void onAfterTableRefreshed(AfterTableRefreshedEvent event) {
+    HashMap<String, Object> properties = new HashMap<>();
+    properties.put("event_type", event.getClass().getSimpleName());
+    properties.put("table_identifier", event.tableIdentifier().toString());
+    transformAndSendEvent(properties);
+  }
+}

runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchConfiguration.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch;
+
+/** Configuration interface for AWS CloudWatch event listener settings. */
+public interface AwsCloudWatchConfiguration {
+  String awsCloudwatchlogGroup();
+
+  String awsCloudwatchlogStream();
+
+  String awsCloudwatchRegion();
+
+  boolean synchronousMode();
+}

runtime/service/src/main/java/org/apache/polaris/service/events/jsonEventListener/aws/cloudwatch/AwsCloudWatchEventListener.java

@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.smallrye.common.annotation.Identifier;
+import jakarta.annotation.PostConstruct;
+import jakarta.annotation.PreDestroy;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import jakarta.ws.rs.core.Context;
+import jakarta.ws.rs.core.SecurityContext;
+import java.time.Clock;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.service.events.jsonEventListener.JsonEventListener;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.cloudwatchlogs.CloudWatchLogsAsyncClient;
+import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupRequest;
+import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogGroupResponse;
+import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamRequest;
+import software.amazon.awssdk.services.cloudwatchlogs.model.CreateLogStreamResponse;
+import software.amazon.awssdk.services.cloudwatchlogs.model.InputLogEvent;
+import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsRequest;
+import software.amazon.awssdk.services.cloudwatchlogs.model.PutLogEventsResponse;
+import software.amazon.awssdk.services.cloudwatchlogs.model.ResourceAlreadyExistsException;
+
+@ApplicationScoped
+@Identifier("aws-cloudwatch")
+public class AwsCloudWatchEventListener extends JsonEventListener {
+  private static final Logger LOGGER = LoggerFactory.getLogger(AwsCloudWatchEventListener.class);
+  private final ObjectMapper objectMapper = new ObjectMapper();
+
+  private CloudWatchLogsAsyncClient client;
+
+  private final String logGroup;
+  private final String logStream;
+  private final Region region;
+  private final boolean synchronousMode;
+  private final Clock clock;
+
+  @Inject CallContext callContext;
+
+  @Context SecurityContext securityContext;
+
+  @Inject
+  public AwsCloudWatchEventListener(AwsCloudWatchConfiguration config, Clock clock) {
+    this.logStream = config.awsCloudwatchlogStream();
+    this.logGroup = config.awsCloudwatchlogGroup();
+    this.region = Region.of(config.awsCloudwatchRegion());
+    this.synchronousMode = config.synchronousMode();
+    this.clock = clock;
+  }
+
+  @PostConstruct
+  void start() {
+    this.client = createCloudWatchAsyncClient();
+    ensureLogGroupAndStream();
+  }
+
+  protected CloudWatchLogsAsyncClient createCloudWatchAsyncClient() {
+    return CloudWatchLogsAsyncClient.builder().region(region).build();
+  }
+
+  private void ensureLogGroupAndStream() {
+    try {
+      CompletableFuture<CreateLogGroupResponse> future =
+          client.createLogGroup(CreateLogGroupRequest.builder().logGroupName(logGroup).build());
+      future.join();
+    } catch (CompletionException e) {
+      if (e.getCause() instanceof ResourceAlreadyExistsException) {
+        LOGGER.debug("Log group {} already exists", logGroup);
+      } else {
+        throw e;
+      }
+    }
+
+    try {
+      CompletableFuture<CreateLogStreamResponse> future =
+          client.createLogStream(
+              CreateLogStreamRequest.builder()
+                  .logGroupName(logGroup)
+                  .logStreamName(logStream)
+                  .build());
+      future.join();
+    } catch (CompletionException e) {
+      if (e.getCause() instanceof ResourceAlreadyExistsException) {
+        LOGGER.debug("Log stream {} already exists", logStream);
+      } else {
+        throw e;
+      }
+    }
+  }
+
+  @PreDestroy
+  void shutdown() {
+    if (client != null) {
+      client.close();
+    }
+  }
+
+  @Override
+  protected void transformAndSendEvent(HashMap<String, Object> properties) {
+    properties.put("realm", callContext.getRealmContext().getRealmIdentifier());
+    properties.put("principal", securityContext.getUserPrincipal().getName());
+    // TODO: Add request ID when it is available
+    String eventAsJson;
+    try {
+      eventAsJson = objectMapper.writeValueAsString(properties);
+    } catch (JsonProcessingException e) {
+      LOGGER.error("Error processing event into JSON string: ", e);
+      return;
+    }
+    InputLogEvent inputLogEvent = createLogEvent(eventAsJson, getCurrentTimestamp());
+    PutLogEventsRequest.Builder requestBuilder =
+        PutLogEventsRequest.builder()
+            .logGroupName(logGroup)
+            .logStreamName(logStream)
+            .logEvents(List.of(inputLogEvent));
+    CompletableFuture<PutLogEventsResponse> future =
+        client
+            .putLogEvents(requestBuilder.build())
+            .whenComplete(
+                (resp, err) -> {
+                  if (err != null) {
+                    LOGGER.error(
+                        "Error writing log to CloudWatch. Event: {}, Error: ", inputLogEvent, err);
+                  }
+                });
+    if (synchronousMode) {
+      future.join();
+    }
+  }
+
+  private long getCurrentTimestamp() {
+    return clock.millis();
+  }
+
+  private InputLogEvent createLogEvent(String eventAsJson, long timestamp) {
+    return InputLogEvent.builder().message(eventAsJson).timestamp(timestamp).build();
+  }
+}

runtime/service/src/main/java/org/apache/polaris/service/quarkus/events/jsonEventListener/aws/cloudwatch/QuarkusAwsCloudWatchConfiguration.java

@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.quarkus.events.jsonEventListener.aws.cloudwatch;
+
+import io.quarkus.runtime.annotations.StaticInitSafe;
+import io.smallrye.config.ConfigMapping;
+import io.smallrye.config.WithDefault;
+import io.smallrye.config.WithName;
+import jakarta.enterprise.context.ApplicationScoped;
+import org.apache.polaris.service.events.jsonEventListener.aws.cloudwatch.AwsCloudWatchConfiguration;
+
+/**
+ * Quarkus-specific configuration interface for AWS CloudWatch event listener integration.
+ *
+ * <p>This interface extends the base {@link AwsCloudWatchConfiguration} and provides
+ * Quarkus-specific configuration mappings for AWS CloudWatch logging functionality.
+ */
+@StaticInitSafe
+@ConfigMapping(prefix = "polaris.event-listener.aws-cloudwatch")
+@ApplicationScoped
+public interface QuarkusAwsCloudWatchConfiguration extends AwsCloudWatchConfiguration {
+
+  /**
+   * Returns the AWS CloudWatch log group name for event logging.
+   *
+   * <p>The log group is a collection of log streams that share the same retention, monitoring, and
+   * access control settings. If not specified, defaults to "polaris-cloudwatch-default-group".
+   *
+   * <p>Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-group}
+   *
+   * @return an Optional containing the log group name, or empty if not configured
+   */
+  @WithName("log-group")
+  @WithDefault("polaris-cloudwatch-default-group")
+  @Override
+  String awsCloudwatchlogGroup();
+
+  /**
+   * Returns the AWS CloudWatch log stream name for event logging.
+   *
+   * <p>A log stream is a sequence of log events that share the same source. Each log stream belongs
+   * to one log group. If not specified, defaults to "polaris-cloudwatch-default-stream".
+   *
+   * <p>Configuration property: {@code polaris.event-listener.aws-cloudwatch.log-stream}
+   *
+   * @return an Optional containing the log stream name, or empty if not configured
+   */
+  @WithName("log-stream")
+  @WithDefault("polaris-cloudwatch-default-stream")
+  @Override
+  String awsCloudwatchlogStream();
+
+  /**
+   * Returns the AWS region where CloudWatch logs should be sent.
+   *
+   * <p>This specifies the AWS region for the CloudWatch service endpoint. The region must be a
+   * valid AWS region identifier. If not specified, defaults to "us-east-1".
+   *
+   * <p>Configuration property: {@code polaris.event-listener.aws-cloudwatch.region}
+   *
+   * @return an Optional containing the AWS region, or empty if not configured
+   */
+  @WithName("region")
+  @WithDefault("us-east-1")
+  @Override
+  String awsCloudwatchRegion();
+
+  /**
+   * Returns the synchronous mode setting for CloudWatch logging.
+   *
+   * <p>When set to "true", log events are sent to CloudWatch synchronously, which may impact
+   * application performance but ensures immediate delivery. When set to "false" (default), log
+   * events are sent asynchronously for better performance.
+   *
+   * <p>Configuration property: {@code polaris.event-listener.aws-cloudwatch.synchronous-mode}
+   *
+   * @return a boolean value indicating the synchronous mode setting
+   */
+  @WithName("synchronous-mode")
+  @WithDefault("false")
+  @Override
+  boolean synchronousMode();
+}