[Connector Builder Cloud] Base Micronaut application for the Connecto…

…r Builder Server (#4756)

.bumpversion.cfg

@@ -48,6 +48,8 @@ replace = "version": "{new_version}"
 
 [bumpversion:file:airbyte-connector-builder-server/Dockerfile]
 
+[bumpversion:file:airbyte-connector-atelier-server/Dockerfile] # We can remove this when we deprecate the old Python server with the Micronaut server
+
 [bumpversion:file:airbyte-connector-builder-server/setup.py]
 search = version="{current_version}"
 replace = version="{new_version}"

.github/workflows/gradle.yml

@@ -543,7 +543,7 @@ jobs:
           # Specify top-level and second-level modules. Note there cannot be a space between the comma.
           path: "/actions-runner/_work/airbyte-platform/airbyte-platform/*/build/test-results/*/*.xml,/actions-runner/_work/airbyte-platform/airbyte-platform/*/*/build/test-results/*/*.xml"
           reporter: java-junit
-          fail-on-error: 'false'
+          fail-on-error: "false"
 
       - name: Upload test results to BuildPulse for flaky test detection
         if: "!cancelled()" # Run this step even when the tests fail. Skip if the workflow is cancelled.
@@ -624,7 +624,7 @@ jobs:
   helm-acceptance-test:
     name: "Platform: Acceptance Tests (Helm)"
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: [ start-helm-acceptance-test-runner ] # required to start the main job when the runner is ready
+    needs: [start-helm-acceptance-test-runner] # required to start the main job when the runner is ready
     runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
     # this is the label of the runner
     timeout-minutes: 90
@@ -760,7 +760,7 @@ jobs:
           name: Platform Helm E2E Test Report
           path: "./*/build/test-results/*/*.xml"
           reporter: java-junit
-          fail-on-error: 'false'
+          fail-on-error: "false"
 
       - uses: actions/upload-artifact@v2
         if: failure()

airbyte-connector-atelier-server/Dockerfile

@@ -0,0 +1,18 @@
+ARG JDK_IMAGE=airbyte/airbyte-base-java-image:1.0
+FROM ${JDK_IMAGE} AS connector-atelier-server
+
+ARG VERSION=0.41.0
+
+ENV APPLICATION airbyte-connector-atelier-server
+ENV VERSION ${VERSION}
+
+WORKDIR /app
+
+# This is automatically unzipped by Docker
+ADD bin/${APPLICATION}-${VERSION}.tar /app
+
+# wait for upstream dependencies to become available before starting server
+ENTRYPOINT ["/bin/bash", "-c", "${APPLICATION}-${VERSION}/bin/${APPLICATION}"]
+
+LABEL io.airbyte.version=0.41.0
+LABEL io.airbyte.name=airbyte/connector-atelier-server

airbyte-connector-atelier-server/README.md

@@ -0,0 +1,57 @@
+# Connector Atelier (aka Builder)
+
+The temporary home for the Micronaut microservice version of the Connector Builder Server which will
+be integrated back into `airbyte-connector-builder-server` upon completion.
+
+## Getting started
+
+Install dependencies, compile, and build the server
+```bash
+./gradlew -p oss airbyte-connector-atelier-server:build
+```
+
+Then run the server (You can also do this w/o build)
+```bash
+./gradlew -p oss airbyte-connector-atelier-server:run
+```
+
+The server is now reachable on localhost:80
+
+## Running the new Micronaut server within Airbyte OSS
+
+In addition to running the standalone server, you can also configure a local instance of Airbyte OSS to
+use the version of the server built with Micronaut.
+
+Replace the `airbyte-connector-builder-server` image name in `oss/docker-compose.yaml`:
+```bash
+airbyte-connector-builder-server:
+  image: airbyte/connector-atelier-server:${VERSION}
+```
+
+Start up your local Airbyte instance:
+```bash
+VERSION=dev docker-compose up   
+```
+
+## OpenAPI generation
+
+Run it via Gradle by running this from the Airbyte project root:
+```bash
+./gradlew -p oss airbyte-connector-atelier-server:generateOpenApiServer
+```
+
+## Migrating the module to `airbyte-connector-builder-server`
+
+For ease of development while migrating to the Micronaut server, we will handle all development on this
+separate module. However, what we really want moving forward is to replace the old module with this new
+one. We're using the word `atelier` (def: a workshop or studio, especially one used by an artist or designer)
+to delineate every instance that should be renamed from atelier to builder because the word is not used
+anywhere else in the codebase. When it comes time to switch to the new server, we will remove the original
+`airbyte-connector-builder-server` module and rename everywhere in the code and every folder where atelier is
+mentioned.
+
+## Changing the used CDK version
+
+TODO: The current `connector-builder-server` and `airbyte-webapp` must stay in sync using the same version of
+the `airbyte-cdk` package. We can specify this as an environment variable in the top-level `.env` file. 
+This has not been implemented yet, but we may also need to implement this in a follow-up change.

airbyte-connector-atelier-server/build.gradle

@@ -0,0 +1,88 @@
+import org.openapitools.generator.gradle.plugin.tasks.GenerateTask
+
+plugins {
+    id "org.openapi.generator" version "5.3.1" // todo: can we upgrade this to a later version?
+    id 'application'
+}
+
+dependencies {
+    // Cloud service dependencies. These are not strictly necessary yet, but likely needed for any full-fledged cloud service
+    implementation libs.bundles.datadog
+    // implementation libs.bundles.temporal  uncomment this when we start using temporal to invoke connector commands
+    implementation libs.sentry.java
+
+    // Micronaut dependencies
+    annotationProcessor platform(libs.micronaut.bom)
+    annotationProcessor libs.bundles.micronaut.annotation.processor
+
+    implementation platform(libs.micronaut.bom)
+    implementation libs.bundles.micronaut
+
+    // OpenAPI code generation dependencies
+    implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2'
+}
+
+mainClassName = 'io.airbyte.connector_builder.MicronautConnectorBuilderServerRunner'
+
+application {
+    mainClass = mainClassName
+    applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
+}
+
+Properties env = new Properties()
+rootProject.file('.env.dev').withInputStream { env.load(it) }
+
+run {
+    // default for running on local machine.
+    env.each { entry ->
+        environment entry.getKey(), entry.getValue()
+    }
+
+    environment 'AIRBYTE_ROLE', System.getenv('AIRBYTE_ROLE')
+    environment 'AIRBYTE_VERSION', env.VERSION
+}
+
+task generateOpenApiServer(type: GenerateTask) {
+    def generatedCodeDir = "$buildDir/generated/api/server"
+
+    inputSpec = "$projectDir/src/main/openapi/openapi.yaml"
+    outputDir = generatedCodeDir
+
+    generatorName = "jaxrs-spec"
+    apiPackage = "io.airbyte.connector_builder.api.generated"
+    invokerPackage = "io.airbyte.connector_builder.api.invoker.generated"
+    modelPackage = "io.airbyte.connector_builder.api.model.generated"
+
+    // Our spec does not have nullable, but if it changes, this would be a gotcha that we would want to avoid
+    configOptions = [
+            dateLibrary                   : "java8",
+            generatePom                   : "false",
+            interfaceOnly                 : "true",
+            /*
+            JAX-RS generator does not respect nullable properties defined in the OpenApi Spec.
+            It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json.
+            The below Jackson annotation is made to only keep non null values in serialized json.
+            We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above.
+            Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details.
+            */
+            additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)",
+    ]
+}
+compileJava.dependsOn tasks.generateOpenApiServer
+
+// Ensures that the generated models are compiled during the build step so they are available for use at runtime
+sourceSets {
+    main {
+        java {
+            srcDirs "$buildDir/generated/api/server/src/gen/java"
+        }
+        resources {
+            srcDir "$projectDir/src/main/openapi/"
+        }
+    }
+}
+
+tasks.named("buildDockerImage") {
+    dependsOn generateOpenApiServer
+    dependsOn copyGeneratedTar
+}

airbyte-connector-atelier-server/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=connector-atelier-server

airbyte-connector-atelier-server/src/main/java/io/airbyte/connector_builder/MicronautConnectorBuilderServerRunner.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.connector_builder;
+
+import io.micronaut.runtime.Micronaut;
+
+/**
+ * Micronaut server responsible for running the Connector Builder Server which is used to service
+ * requests to build and test low-code connector manifests.
+ *
+ * Injected object looks unused but they are not
+ */
+public class MicronautConnectorBuilderServerRunner {
+
+  public static void main(final String[] args) {
+    Micronaut.build(args)
+        .mainClass(MicronautConnectorBuilderServerRunner.class)
+        .start();
+  }
+
+}

airbyte-connector-atelier-server/src/main/java/io/airbyte/connector_builder/controllers/ConnectorBuilderController.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+ */
+
+package io.airbyte.connector_builder.controllers;
+
+import io.airbyte.connector_builder.api.generated.V1Api;
+import io.airbyte.connector_builder.api.model.generated.ResolveManifest;
+import io.airbyte.connector_builder.api.model.generated.ResolveManifestRequestBody;
+import io.airbyte.connector_builder.api.model.generated.StreamRead;
+import io.airbyte.connector_builder.api.model.generated.StreamReadPages;
+import io.airbyte.connector_builder.api.model.generated.StreamReadRequestBody;
+import io.airbyte.connector_builder.api.model.generated.StreamReadSlices;
+import io.airbyte.connector_builder.api.model.generated.StreamsListRead;
+import io.airbyte.connector_builder.api.model.generated.StreamsListReadStreams;
+import io.airbyte.connector_builder.api.model.generated.StreamsListRequestBody;
+import io.micronaut.context.annotation.Context;
+import io.micronaut.http.MediaType;
+import io.micronaut.http.annotation.Controller;
+import io.micronaut.http.annotation.Post;
+import io.micronaut.scheduling.TaskExecutors;
+import io.micronaut.scheduling.annotation.ExecuteOn;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Micronaut controller that defines the behavior for all endpoints related to building and testing
+ * low-code connectors using the Connector Builder from the Airbyte web application.
+ */
+@Controller("/v1")
+@Context
+public class ConnectorBuilderController implements V1Api {
+
+  public ConnectorBuilderController() {
+    // Placeholder for now. We just return dummy responses for the base server right now, but we should
+    // define any helper handlers here
+  }
+
+  @Override
+  @Post(uri = "/streams/list",
+        produces = MediaType.APPLICATION_JSON)
+  @ExecuteOn(TaskExecutors.IO)
+  public StreamsListRead listStreams(final StreamsListRequestBody streamsListRequestBody) {
+    final StreamsListReadStreams survivors_stream = new StreamsListReadStreams();
+    survivors_stream.setName("survivors");
+    survivors_stream.setUrl("https://the-last-of-us.com/v1/survivors");
+    final StreamsListReadStreams locations_stream = new StreamsListReadStreams();
+    locations_stream.setName("locations");
+    locations_stream.setUrl("https://the-last-of-us.com/v1/locations");
+
+    final StreamsListRead streamsResponse = new StreamsListRead();
+    streamsResponse.setStreams(List.of(survivors_stream, locations_stream));
+    return streamsResponse;
+  }
+
+  @Override
+  @Post(uri = "/stream/read",
+        produces = MediaType.APPLICATION_JSON)
+  @ExecuteOn(TaskExecutors.IO)
+  public StreamRead readStream(final StreamReadRequestBody streamReadRequestBody) {
+    final HashMap<String, String> recordOne = new HashMap<>();
+    recordOne.put("name", "Joel Miller");
+    final HashMap<String, String> recordTwo = new HashMap<>();
+    recordTwo.put("name", "Ellie Williams");
+
+    final StreamReadPages pages = new StreamReadPages();
+    pages.setRecords(List.of(recordOne, recordTwo));
+    final StreamReadSlices slices = new StreamReadSlices();
+    slices.setPages(List.of(pages));
+    final StreamRead readResponse = new StreamRead();
+    readResponse.setSlices(List.of(slices));
+    return readResponse;
+  }
+
+  @Override
+  @Post(uri = "/manifest/resolve",
+        produces = MediaType.APPLICATION_JSON)
+  @ExecuteOn(TaskExecutors.IO)
+  public ResolveManifest resolveManifest(final ResolveManifestRequestBody resolveManifestRequestBody) {
+    final ResolveManifest resolvedManifest = new ResolveManifest();
+    resolvedManifest.setManifest("Resolved a manifest");
+    return resolvedManifest;
+  }
+
+}