.buildkite/pipeline.yml

@@ -0,0 +1,11 @@
+steps:
+  - label: ":java: :elasticsearch: Elasticsearch Java API client DRA - {{matrix.workflow}}"
+    agents:
+      provider: "gcp"
+    branches: [ "main", "7.17", "8.17", "8.18", "8.x", "9.0" ]
+    matrix:
+      setup:
+        workflow:
+          - "snapshot"
+          - "staging"
+    command: ".ci/release_dra.sh {{matrix.workflow}}"

.buildkite/release_central.yml

@@ -0,0 +1,23 @@
+agents:
+  provider: "gcp"
+
+steps:
+  - label: ":java: :elasticsearch: Elasticsearch Java API client - Release"
+    key: "release"
+    command: ".ci/release_central.sh"
+    artifact_paths:
+      - ".ci/output/repository/**/*"
+      - "signed-artifacts/**/*"
+  - wait: ~
+  - label: "Publish S3 Artifacts :s3:"
+    trigger: unified-release-publish-s3-artifacts
+    key: publish-s3-service
+    build:
+      env:
+        DESTINATION_PATH: "s3://artifacts.elastic.co/javadoc/co/elastic/clients/elasticsearch-java/"
+        EXTRACT_ARTIFACT: "${VERSION}/elasticsearch-java-${VERSION}-javadoc.jar"
+
+#notify:
+#  - slack: "#devtools-notify"
+#    # skip slack messages if no failures and dry-run mode
+#    if: 'build.state != "passed" && build.env("dry_run") == "false"'

.ci/Dockerfile

@@ -20,22 +20,42 @@
 # Build a Docker image that has Gradle and all project dependencies ready.
 # Needs to be run with .git and java-client/src mounted under /elasticsearch-java.
 
-FROM openjdk:15
+FROM eclipse-temurin:17
+
+# Default UID/GID to 1000
+# it can be overridden at build time
+ARG BUILDER_UID=1000
+ARG BUILDER_GID=1000
+
+# Create group and user if needed
+RUN (getent group ${BUILDER_GID} || groupadd -g ${BUILDER_GID} elastic) \
+    && (getent passwd ${BUILDER_UID} || useradd -u ${BUILDER_UID} -g ${BUILDER_GID} elastic) \
+    && mkdir -p /elasticsearch-java/build \
+    && chown -R ${BUILDER_UID}:${BUILDER_GID} /elasticsearch-java \
+    && chown -R ${BUILDER_UID}:${BUILDER_GID} /home
 
 WORKDIR /elasticsearch-java/
+USER ${BUILDER_UID}:${BUILDER_GID}
 
 # Download and initialize Gradle
-COPY gradlew ./
-COPY gradle ./gradle
+COPY --chown=$BUILDER_UID:$BUILDER_GID gradlew ./
+COPY --chown=$BUILDER_UID:$BUILDER_GID gradle ./gradle
 RUN ./gradlew
 
-COPY LICENSE.txt NOTICE.txt ./
+COPY --chown=$BUILDER_UID:$BUILDER_GID LICENSE.txt NOTICE.txt ./
 
-# Prefetch dependencies
-COPY build.gradle.kts settings.gradle.kts ./
-COPY buildSrc ./buildSrc/
-COPY config ./config/
-COPY java-client/build.gradle.kts ./java-client/
+# Prefetch and cache dependencies
+COPY --chown=$BUILDER_UID:$BUILDER_GID build.gradle.kts settings.gradle.kts ./
+COPY --chown=$BUILDER_UID:$BUILDER_GID buildSrc ./buildSrc/
+COPY --chown=$BUILDER_UID:$BUILDER_GID config ./config/
+COPY --chown=$BUILDER_UID:$BUILDER_GID java-client/build.gradle.kts ./java-client/
 RUN ./gradlew resolveDependencies
 
+# Note: trailing '*' avoids failing if the file doesn't exist
+COPY --chown=$BUILDER_UID:$BUILDER_GID gradle.properties* ./
+
+# Build artifacts. Expects these mounted directories:
+# /elasticsearch-java/.git - git index (read-only)
+# /elasticsearch-java/java-client/src - source files (read-only)
+# /elasticsearch-java/build - output directory (read-write)
 ENTRYPOINT ["./gradlew"]

.ci/configure_signing.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#see https://central.sonatype.org/publish/publish-gradle/#distributing-your-public-key
+
+set -euo pipefail
+
+if grep -sq "signing.keyId" gradle.properties; then
+  # Keys already present
+  exit 0
+fi
+
+mkdir -p /tmp/secured
+keyring_file="/tmp/secured/keyring.gpg"
+
+vault_path="kv/ci-shared/release-eng/team-release-secrets/elasticsearch-java"
+
+vault kv get --field="keyring" $vault_path/gpg | base64 -d > $keyring_file
+## NOTE: passphase is the name of the field.
+signing_password=$(vault kv get --field="passphase" $vault_path/gpg)
+signing_key=$(vault kv get --field="key_id" $vault_path/gpg)
+
+maven_username=$(vault kv get --field="username" $vault_path/maven_central)
+maven_password=$(vault kv get --field="password" $vault_path/maven_central)
+
+cat >> gradle.properties <<EOF
+signing.keyId=${signing_key: -8}
+signing.password=${signing_password}
+signing.secretKeyRingFile=${keyring_file}
+
+sonatypeUsername=${maven_username}
+sonatypePassword=${maven_password}
+EOF

.ci/make.sh

@@ -42,7 +42,9 @@
 # Bootstrap
 # ------------------------------------------------------- #
 
-script_path=$(dirname "$(realpath -s "$0")")
+set -euo pipefail
+
+script_path=$(dirname "$(realpath "$0")")
 repo=$(realpath "$script_path/../")
 
 # shellcheck disable=SC1090
@@ -51,12 +53,12 @@ TASK=$1
 TASK_ARGS=()
 VERSION=$2
 STACK_VERSION=$VERSION
-set -euo pipefail
 
 product="elastic/elasticsearch-java"
 output_folder=".ci/output"
 codegen_folder=".ci/output"
 OUTPUT_DIR="$repo/${output_folder}"
+WORKFLOW="${WORKFLOW-staging}"
 mkdir -p "$OUTPUT_DIR"
 
 echo -e "\033[34;1mINFO:\033[0m PRODUCT ${product}\033[0m"
@@ -80,6 +82,15 @@ case $CMD in
             exit 1
         fi
         echo -e "\033[36;1mTARGET: assemble artefact $VERSION\033[0m"
+        TASK=assemble
+        TASK_ARGS=("$VERSION" "$output_folder")
+        ;;
+    release)
+        if [ -z "$VERSION" ]; then
+            echo -e "\033[31;1mTARGET: release -> missing version parameter\033[0m"
+            exit 1
+        fi
+        echo -e "\033[36;1mTARGET: release artefact $VERSION\033[0m"
         TASK=release
         TASK_ARGS=("$VERSION" "$output_folder")
         ;;
@@ -110,7 +121,7 @@ case $CMD in
         TASK_ARGS=("$VERSION" "$codegen_folder")
         ;;
     bump)
-        if [ -v $VERSION ]; then
+        if [ -z "$VERSION" ]; then
             echo -e "\033[31;1mTARGET: bump -> missing version parameter\033[0m"
             exit 1
         fi
@@ -137,9 +148,11 @@ output_mount="-v $repo/.ci/output:/elasticsearch-java/build"
 build_image() {
   echo -e "\033[34;1mINFO: building $product container\033[0m"
 
-  docker build --file .ci/Dockerfile --tag $docker_image \
-    --build-arg USER_ID="$(id -u)" \
-    --build-arg GROUP_ID="$(id -g)" .
+  docker build \
+    --file .ci/Dockerfile \
+    --tag $docker_image \
+    --build-arg BUILDER_UID="$(id -u)" \
+    --build-arg BUILDER_GID="$(id -g)" .
 }
 
 # ------------------------------------------------------- #
@@ -148,8 +161,16 @@ build_image() {
 
 if [[ "$CMD" == "assemble" ]]; then
   rm -rf .ci/output/repository
+
+  if [[ "$WORKFLOW" == 'snapshot' ]]; then
+    assemble_version="$VERSION-SNAPSHOT"
+  else
+    assemble_version="$VERSION"
+  fi
+
   build_image
-  docker run --rm --env VERSION=$VERSION \
+  echo -e "\033[34;1mINFO:\033[0m Building version ${assemble_version}\033[0m"
+  docker run --rm --env VERSION=$assemble_version -u "$(id -u)" \
     $git_mount $src_mount $output_mount \
     $docker_image \
     publishForReleaseManager
@@ -158,15 +179,38 @@ if [[ "$CMD" == "assemble" ]]; then
 	  if [[ -n ${DEPENDENCIES_REPORTS_DIR+x} ]]; then
 	    cp .ci/output/release/dependencies.csv "$DEPENDENCIES_REPORTS_DIR"/"$DEPENDENCIES_REPORT"
     fi
-		echo -e "\033[32;1mTARGET: successfully assembled client version $VERSION\033[0m"
+		echo -e "\033[32;1mTARGET: successfully assembled client version $assemble_version\033[0m"
 	else
 		echo -e "\033[31;1mTARGET: assemble failed, empty workspace!\033[0m"
 		exit 1
 	fi
 fi
 
+if [[ "$CMD" == "release" ]]; then
+  rm -rf .ci/output/repository
+  rm -rf signed-artifacts
+  build_image
+  echo -e "\033[34;1mINFO:\033[0m Building version ${VERSION}\033[0m"
+
+  if [[ "$DRY_RUN" = "true" ]]; then
+    echo "Dry run: building and publishing to the local repository"
+    gradle_task="publishForReleaseManager"
+  else
+    echo "Releasing to Maven snapshot repo"
+    gradle_task="publishToSonatype closeAndReleaseStagingRepositories"
+  fi
+  docker run --rm --env VERSION=$VERSION -u "$(id -u)" \
+    $git_mount $src_mount $output_mount \
+    -v /tmp/secured:/tmp/secured \
+    $docker_image \
+    $gradle_task
+
+  mkdir -p signed-artifacts/$VERSION
+  cp .ci/output/repository/co/elastic/clients/elasticsearch-java/$VERSION/elasticsearch-java-${VERSION}-javadoc.jar signed-artifacts/$VERSION/elasticsearch-java-${VERSION}-javadoc.jar
+fi
+
 if [[ "$CMD" == "bump" ]]; then
-    echo "TODO"
+  echo $VERSION > config/version.txt
 fi
 
 if [[ "$CMD" == "codegen" ]]; then

.ci/release_central.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+#see https://central.sonatype.org/publish/publish-gradle/#distributing-your-public-key
+
+set -euo pipefail
+
+.ci/configure_signing.sh
+
+.ci/make.sh release $VERSION

.ci/release_dra.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+#
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -euo pipefail
+
+WORKFLOW=$1
+STACK_VERSION=$(cat config/version.txt)
+
+# Branch: BRANCH env var > Buildkite's BUILDKITE_BRANCH var > minor version
+BRANCH=${BRANCH:-${BUILDKITE_BRANCH:-${STACK_VERSION%.*}}}
+
+if [[ "$BRANCH" = "main" && "$WORKFLOW" = "staging" ]]; then
+  echo "No staging build for the main branch - skipping"
+  exit 0
+fi
+
+# Add snapshot suffix if needed
+BUILD_VERSION=$STACK_VERSION
+[ "$WORKFLOW" = "snapshot" ] && BUILD_VERSION="${BUILD_VERSION}-SNAPSHOT"
+
+echo "Releasing version $STACK_VERSION, branch: $BRANCH, workflow: $WORKFLOW"
+
+DRA_CREDS=$(vault kv get -field=data -format=json kv/ci-shared/release/dra-role)
+
+.ci/make.sh assemble "$BUILD_VERSION"
+
+# set required permissions on artifacts and directory
+chmod -R a+r $PWD/.ci/output/*
+chmod -R a+w $PWD/.ci/output
+# Artifacts should be generated
+if [ "$WORKFLOW" = "snapshot" ]; then
+  docker run --rm \
+         --name release-manager \
+         -e VAULT_ADDR="$(echo "$DRA_CREDS" | jq -r '.vault_addr')" \
+         -e VAULT_ROLE_ID="$(echo "$DRA_CREDS" | jq -r '.role_id')" \
+         -e VAULT_SECRET_ID="$(echo "$DRA_CREDS" | jq -r '.secret_id')" \
+         --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+         docker.elastic.co/infra/release-manager:latest \
+         cli collect \
+         --project elasticsearch-java \
+         --branch "$BRANCH" \
+         --commit "$(git rev-parse HEAD)" \
+         --workflow "$WORKFLOW" \
+         --version "$STACK_VERSION" \
+         --artifact-set main
+fi
+if [ "$WORKFLOW" = "staging" ]; then
+  docker run --rm \
+         --name release-manager \
+         -e VAULT_ADDR="$(echo "$DRA_CREDS" | jq -r '.vault_addr')" \
+         -e VAULT_ROLE_ID="$(echo "$DRA_CREDS" | jq -r '.role_id')" \
+         -e VAULT_SECRET_ID="$(echo "$DRA_CREDS" | jq -r '.secret_id')" \
+         --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+         docker.elastic.co/infra/release-manager:latest \
+         cli collect \
+         --project elasticsearch-java \
+         --branch "$BRANCH" \
+         --commit "$(git rev-parse HEAD)" \
+         --workflow "$WORKFLOW" \
+         --version "$STACK_VERSION" \
+         --artifact-set main
+fi

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -6,7 +6,7 @@ body:
     attributes:
       value: |
         Please first search existing issues for the feature you are requesting. It may already exist, even as a closed issue.
-        If it exists, please add a comment: along with subcribing you to the issue, this will help us prioritize features.
+        If it exists, please add a comment: along with subscribing you to the issue, this will help us prioritize features.
   - type: textarea
     id: description
     attributes:

.github/workflows/backport.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   backport:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     name: Backport
     steps:
       - name: Backport

.github/workflows/checkstyle.yml

@@ -6,7 +6,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java-version: [ 11 ]
+        java-version: [ 17 ]
     steps:
       - uses: actions/checkout@v2