.gitlab-ci.yml

include:
  - local: ".gitlab/one-pipeline.locked.yml"
  - local: ".gitlab/benchmarks.yml"
  - local: ".gitlab/macrobenchmarks.yml"
  - local: ".gitlab/exploration-tests.yml"
  - local: ".gitlab/ci-visibility-tests.yml"

stages:
  - build
  - publish
  - shared-pipeline
  - benchmarks
  - macrobenchmarks
  - tests
  - test-summary
  - exploration-tests
  - ci-visibility-tests
  - generate-signing-key

variables:
  # Gitlab runner features; see https://docs.gitlab.com/runner/configuration/feature-flags.html
  # Fold and time all script sections
  FF_SCRIPT_SECTIONS: 1

  REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
  BUILD_JOB_NAME: "build"
  DEPENDENCY_CACHE_POLICY: pull
  BUILD_CACHE_POLICY: pull
  GRADLE_VERSION: "8.14.4" # must match gradle-wrapper.properties
  MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
  GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/"
  BUILDER_IMAGE_VERSION_PREFIX: "v26.02-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "v25.05-")
  REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations"
  DEFAULT_TEST_JVMS: /^(8|11|17|21|25|stable)$/ # the latest "stable" version is 26
  PROFILE_TESTS:
    description: "Enable profiling of tests"
    value: "false"
  NON_DEFAULT_JVMS:
    description: "Enable tests on JVMs that are not the default"
    value: "false"
  RUN_FLAKY_TESTS:
    description: "Enable flaky tests"
    value: "false"

# trigger new commit cancel
workflow:
  auto_cancel:
    on_new_commit: interruptible
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
      auto_cancel:
        on_new_commit: none
    - if: '$CI_COMMIT_BRANCH =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      auto_cancel:
        on_new_commit: none
    - when: always

.test_matrix: &test_matrix
  - testJvm: &test_jvms
      - "8"
      - "11"
      - "17"
      - "21"
      - "25"
      - "semeru11"
      - "oracle8"
      - "zulu8"
      - "semeru8"
      - "ibm8"
      - "zulu11"
      - "semeru17"
      - "stable"
    CI_SPLIT: ["1/1"]

# Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
# These blocks emulate "parallel" by including it in the matrix
.test_matrix_2: &test_matrix_2
  - testJvm: *test_jvms
    CI_SPLIT: ["1/2", "2/2"]

.test_matrix_4: &test_matrix_4
  - testJvm: *test_jvms
    CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

.test_matrix_6: &test_matrix_6
  - testJvm: *test_jvms
    CI_SPLIT: ["1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]

.test_matrix_8: &test_matrix_8
  - testJvm: *test_jvms
    CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"]

.test_matrix_12: &test_matrix_12
  - testJvm: *test_jvms
    CI_SPLIT: [ "1/12", "2/12", "3/12", "4/12", "5/12", "6/12", "7/12", "8/12", "9/12", "10/12", "11/12", "12/12" ]

.master_only: &master_only
  - if: $CI_COMMIT_BRANCH == "master"
    when: on_success

default:
  tags: [ "arch:amd64" ]
  interruptible: true

.set_datadog_api_keys: &set_datadog_api_keys
  - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.DATADOG_API_KEY_PROD --with-decryption --query "Parameter.Value" --out text)

# CI_NODE_INDEX and CI_NODE_TOTAL are 1-indexed and not always set. These steps normalize the numbers for jobs
.normalize_node_index: &normalize_node_index
  - if [ "$CI_NO_SPLIT" == "true" ] ; then CI_NODE_INDEX=1; CI_NODE_TOTAL=1; fi # A job uses parallel but doesn't intend to split by index
  - if [ -n "$CI_SPLIT" ]; then CI_NODE_INDEX="${CI_SPLIT%%/*}"; CI_NODE_TOTAL="${CI_SPLIT##*/}"; fi
  - echo "CI_NODE_INDEX=$CI_NODE_INDEX, CI_NODE_TOTAL=${CI_NODE_TOTAL}"

.cgroup_info: &cgroup_info
  - source .gitlab/gitlab-utils.sh
  - gitlab_section_start "cgroup-info" "cgroup info"
  - .gitlab/cgroup-info.sh
  - gitlab_section_end "cgroup-info"

.container_info: &container_info
  - |
    # Containers and processes are limited to a 1-hour window, and expire after 36 hours
    if [ -n "$CI_JOB_STARTED_AT" ]; then
      FROM_EPOCH_MS=$(date -d "$CI_JOB_STARTED_AT" +%s)000
      TO_EPOCH_MS=$((FROM_EPOCH_MS + 3600000))
      TIME_PARAMS="from_ts=${FROM_EPOCH_MS}&to_ts=${TO_EPOCH_MS}&"
    else
      TIME_PARAMS=""
    fi

    echo -e "${TEXT_BOLD}${TEXT_YELLOW}Runner dashboard, these are live (limited to a 1-hour window, and expire after 36 hours)${TEXT_CLEAR}"
    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"
    echo -e "${TEXT_BOLD}${TEXT_YELLOW}   Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false"

.gitlab_base_ref_params: &gitlab_base_ref_params
  - |
    export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh)
    if [[ -n "$GIT_BASE_REF" ]]; then
      export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF"
    else
      echo "Failed to find base ref for PR" >&2
    fi

.gradle_build: &gradle_build
  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: build
  variables:
    MAVEN_OPTS: "-Xms256M -Xmx1024M"
    GRADLE_WORKERS: 6
    GRADLE_MEMORY_MIN: 1G
    GRADLE_MEMORY_MAX: 4G
    KUBERNETES_CPU_REQUEST: 10
    KUBERNETES_MEMORY_REQUEST: 20Gi
    KUBERNETES_MEMORY_LIMIT: 20Gi
    CACHE_TYPE: "lib" #default
    FF_USE_FASTZIP: "true"
    CACHE_COMPRESSION_LEVEL: "slowest"

    RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes
    GIT_SUBMODULE_STRATEGY: normal
    GIT_SUBMODULE_DEPTH: 1
  cache:
    - key: dependency-$CACHE_TYPE # Dependencies cache
      paths:
        # Cached dependencies and wrappers for Gradle and Maven:
        - .gradle/wrapper
        - .gradle/caches
        - .gradle/notifications
        - .mvn/caches
      policy: $DEPENDENCY_CACHE_POLICY
      unprotect: true
      fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
        - 'dependency-base'
        - 'dependency-lib'
    - key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
      paths:
        - .gradle/caches/$GRADLE_VERSION
        - .gradle/$GRADLE_VERSION/executionHistory
        - workspace
      policy: $BUILD_CACHE_POLICY
      unprotect: true
  before_script:
    - source .gitlab/gitlab-utils.sh
    - *container_info
    # Akka token added to SSM from https://account.akka.io/token
    - export ORG_GRADLE_PROJECT_akkaRepositoryToken=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.akka_repo_token --with-decryption --query "Parameter.Value" --out text)
    - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY
    - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY
    - |
      JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,)
      cat >> gradle.properties <<EOF
      org.gradle.java.installations.auto-detect=false
      org.gradle.java.installations.auto-download=false
      org.gradle.java.installations.fromEnv=$JAVA_HOMES
      EOF
    - mkdir -p .gradle
    - export GRADLE_USER_HOME=$(pwd)/.gradle
    # replace maven central part by MAVEN_REPOSITORY_PROXY in .mvn/wrapper/maven-wrapper.properties
    - sed -i "s|https://repo.maven.apache.org/maven2/|$MAVEN_REPOSITORY_PROXY|g" .mvn/wrapper/maven-wrapper.properties
    - mkdir -p .mvn/caches
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - export GRADLE_ARGS=" --build-cache --stacktrace --no-daemon --parallel --max-workers=$GRADLE_WORKERS"
    - *normalize_node_index
    # for weird reasons, Gradle will always "chmod 700" the .gradle folder
    # with Gitlab caching, .gradle is always owned by root and thus Gradle's chmod invocation fails
    # This dance is a hack to have .gradle owned by the Gitlab runner user
    - gitlab_section_start "gradle-dance" "Fix .gradle directory permissions"
    - cp -r .gradle .gradle-copy
    - rm -rf .gradle
    - mv .gradle-copy .gradle
    - ls -la
    - gitlab_section_end "gradle-dance"
  after_script:
    - *cgroup_info
    - *container_info

# Check and fail early if maven central credentials are incorrect. When a new token is generated
# on the central publisher portal, it invalidates the old one. This check prevents going further.
# See https://datadoghq.atlassian.net/wiki/x/Oog5OgE
maven-central-pre-release-check:
  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: .pre
  rules:
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
      allow_failure: false
  script:
    - |
      MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
      MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
      # See https://central.sonatype.org/publish/publish-portal-api/
      # 15e0cbbb-deff-421e-9e02-296a24d0cada is deployment, any deployment id listed in central  work, the idea is to check whether the token can authenticate
      curl --request POST --include --fail https://central.sonatype.com/api/v1/publisher/status?id=15e0cbbb-deff-421e-9e02-296a24d0cada --header "Authorization: Bearer $(printf "$MAVEN_CENTRAL_USERNAME:$MAVEN_CENTRAL_PASSWORD" | base64)"
      if [ $? -ne 0 ]; then
        echo "Failed to authenticate against central. Check credentials, see https://datadoghq.atlassian.net/wiki/x/Oog5OgE"
        exit 1
      fi

dd-octo-sts-pre-release-check:
  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
  stage: .pre
  tags: [ "arch:amd64" ]
  id_tokens:
    DDOCTOSTS_ID_TOKEN:
      aud: dd-octo-sts
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
      allow_failure: false
  before_script:
    - dd-octo-sts version
    - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release
    - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > test-github-token.txt
  script:
    - gh auth login --with-token < test-github-token.txt
    - gh auth status
  after_script:
    - dd-octo-sts revoke -t $(cat test-github-token.txt)
  retry:
    max: 2
    when: always

build:
  needs:
    - job: maven-central-pre-release-check
      optional: true
    - job: dd-octo-sts-pre-release-check
      optional: true
  extends: .gradle_build
  variables:
    BUILD_CACHE_POLICY: push
    CACHE_TYPE: "lib"
    DEPENDENCY_CACHE_POLICY: pull
  script:
    - if [ $CI_PIPELINE_SOURCE == "schedule" ] ; then ./gradlew resolveAndLockAll --write-locks $GRADLE_ARGS; fi
    - ./gradlew --version
    - ./gradlew clean :dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar -PskipTests $GRADLE_ARGS
    - echo UPSTREAM_TRACER_VERSION=$(java -jar workspace/dd-java-agent/build/libs/*.jar) >> upstream.env
    - echo "BUILD_JOB_NAME=$CI_JOB_NAME" >> build.env
    - echo "BUILD_JOB_ID=$CI_JOB_ID" >> build.env
  artifacts:
    when: always
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'
      - 'upstream.env'
      - '.gradle/daemon/*/*.out.log'
    reports:
      dotenv: build.env

build_tests:
  extends: .gradle_build
  variables:
    BUILD_CACHE_POLICY: push
    DEPENDENCY_CACHE_POLICY: pull
  parallel:
    matrix:
      - GRADLE_TARGET: ":baseTest"
        CACHE_TYPE: "base"
      - GRADLE_TARGET: ":profilingTest"
        CACHE_TYPE: "profiling"
      - GRADLE_TARGET: ":instrumentationTest"
        CACHE_TYPE: "inst"
      - GRADLE_TARGET: ":instrumentationLatestDepTest"
        CACHE_TYPE: "latestdep"
      - GRADLE_TARGET: ":smokeTest"
        CACHE_TYPE: "smoke"
        MAVEN_OPTS: "-Xms256M -Xmx1024M"
  script:
    - *gitlab_base_ref_params
    - ./gradlew --version
    - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS

populate_dep_cache:
  extends: build_tests
  variables:
    BUILD_CACHE_POLICY: pull
    DEPENDENCY_CACHE_POLICY: push
  rules:
    - if: '$POPULATE_CACHE'
      when: on_success
    - when: manual
      allow_failure: true
  parallel:
    matrix:
      - GRADLE_TARGET: ":dd-java-agent:shadowJar :dd-trace-api:jar :dd-trace-ot:shadowJar"
        CACHE_TYPE: "lib"
      - GRADLE_TARGET: ":baseTest"
        CACHE_TYPE: "base"
      - GRADLE_TARGET: ":profilingTest"
        CACHE_TYPE: "profiling"
      - GRADLE_TARGET: ":instrumentationTest"
        CACHE_TYPE: "inst"
      - GRADLE_TARGET: ":instrumentationLatestDepTest"
        CACHE_TYPE: "latestdep"
      - GRADLE_TARGET: ":smokeTest"
        CACHE_TYPE: "smoke"

publish-artifacts-to-s3:
  image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
  stage: publish
  needs: [ build ]
  script:
    - source upstream.env
    - export VERSION="${UPSTREAM_TRACER_VERSION%~*}" # remove ~githash from the end of version
    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-java-agent.jar
    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-api.jar
    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_COMMIT_REF_NAME}/dd-trace-ot.jar
    - aws s3 cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar
    - aws s3 cp workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-api.jar
    - aws s3 cp workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar s3://dd-trace-java-builds/${CI_PIPELINE_ID}/dd-trace-ot.jar
    - |
      cat << EOF > links.json 
        {
          "S3 Links": [
            {
              "external_link": {
                "label": "Public Link to dd-java-agent.jar",
                "url": "https://s3.us-east-1.amazonaws.com/dd-trace-java-builds/${CI_PIPELINE_ID}/dd-java-agent.jar"
              }
            }
          ]
        }
      EOF
  artifacts:
    reports:
      annotations:
        - links.json


spotless:
  extends: .gradle_build
  stage: tests
  needs: []
  variables:
    GRADLE_MEMORY_MAX: 6G
  script:
    - ./gradlew --version
    - ./gradlew spotlessCheck $GRADLE_ARGS

check-instrumentation-naming:
  extends: .gradle_build
  stage: tests
  needs: [ ]
  script:
    - ./gradlew --version
    - ./gradlew checkInstrumentationNaming

config-inversion-linter:
  extends: .gradle_build
  stage: tests
  needs: []
  script:
    - ./gradlew --version
    - ./gradlew logEnvVarUsages checkEnvironmentVariablesUsage checkConfigStrings

test_published_artifacts:
  extends: .gradle_build
  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}7 # Needs Java7 for some tests
  stage: tests
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  script:
    - mvn_local_repo=$(./mvnw help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)
    - rm -rf "${mvn_local_repo}/com/datadoghq"
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms2G -Xmx2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - ./gradlew publishToMavenLocal $GRADLE_ARGS
    - cd test-published-dependencies
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms1G -Xmx1G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
    - ./gradlew --version
    - ./gradlew check --info $GRADLE_ARGS
  after_script:
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./check_reports

.check_job:
  extends: .gradle_build
  needs: [ build ]
  stage: tests
  variables:
    CACHE_TYPE: "lib"
  script:
    - *gitlab_base_ref_params
    - ./gradlew --version
    - ./gradlew $GRADLE_TARGET -x spotlessCheck $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh --destination ./check_reports --move
    - .gitlab/collect_results.sh
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./check_reports
      - ./results
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml
  retry:
    max: 2
    when:
      - unknown_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - unmet_prerequisites
      - scheduler_failure
      - data_integrity_failure

check_build_src:
  extends: .check_job
  needs: []
  variables:
    GRADLE_TARGET: ":buildSrc:build"

check_base:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":baseCheck"

check_inst:
  extends: .check_job
  parallel: 4
  variables:
    GRADLE_TARGET: ":instrumentationCheck"
    CACHE_TYPE: "inst"

check_smoke:
  extends: .check_job
  parallel: 4
  variables:
    GRADLE_TARGET: ":smokeCheck"
    CACHE_TYPE: "smoke"

check_profiling:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":profilingCheck"

check_debugger:
  extends: .check_job
  variables:
    GRADLE_TARGET: ":debuggerCheck"

muzzle:
  extends: .gradle_build
  needs: [ build_tests ]
  stage: tests
  parallel:
    matrix:
      - CI_SPLIT: ["1/8", "2/8", "3/8", "4/8", "5/8", "6/8", "7/8", "8/8"]
  variables:
    CACHE_TYPE: "inst"
  script:
    - export SKIP_BUILDSCAN="true"
    - ./gradlew --version
    - ./gradlew :runMuzzle -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - *set_datadog_api_keys
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - .gitlab/collect_results.sh
    - .gitlab/upload_ciapp.sh $CACHE_TYPE
    - gitlab_section_end "collect-reports"
  artifacts:
    when: always
    paths:
      - ./reports
      - ./results
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml

muzzle-dep-report:
  extends: .gradle_build
  needs: [ build_tests ]
  stage: tests
  variables:
    CACHE_TYPE: "inst"
  script:
    - export SKIP_BUILDSCAN="true"
    - ./gradlew --version
    - ./gradlew generateMuzzleReport muzzleInstrumentationReport $GRADLE_ARGS
  after_script:
    - *container_info
    - *cgroup_info
    - .gitlab/collect_muzzle_deps.sh
  artifacts:
    when: always
    paths:
      - ./reports
      - '.gradle/daemon/*/*.out.log'

# In Gitlab, DD_* variables are set because the build runner is instrumented with Datadog telemetry
# To have a pristine environment for the tests, these variables are saved before the test run and restored afterwards
.prepare_test_env: &prepare_test_env
  - export gitlabVariables=("DD_SERVICE" "DD_ENTITY_ID" "DD_SITE" "DD_ENV" "DD_DATACENTER" "DD_PARTITION" "DD_CLOUDPROVIDER")
  - '[ ! -e pretest.env ] || rm pretest.env'
  - |
    for VARIABLE in "${gitlabVariables[@]}"
    do
      echo "export $VARIABLE=${!VARIABLE}" >> pretest.env
      unset "$VARIABLE"
    done

.restore_pretest_env: &restore_pretest_env
  - source pretest.env

.test_job:
  extends: .gradle_build
  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
  tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
  needs: [ build_tests ]
  stage: tests
  variables:
    GRADLE_PARAMS: "-PskipFlakyTests"
    CONTINUE_ON_FAILURE: "false"
    TESTCONTAINERS_CHECKS_DISABLE: "true"
    TESTCONTAINERS_RYUK_DISABLED: "true"
    TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/"
    JETTY_AVAILABLE_PROCESSORS: 4 # Jetty incorrectly calculates processor count in containers
  rules:
    - if: $testJvm =~ $DEFAULT_TEST_JVMS
      when: on_success
    - if: $NON_DEFAULT_JVMS == "true"
      when: on_success
    - if: $CI_COMMIT_BRANCH == "master"
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^mq-working-branch-/'
      when: on_success
    - if: '$CI_COMMIT_BRANCH =~ /^gh-readonly-queue/'
      when: on_success
  script:
    - *gitlab_base_ref_params
    - >
      if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ];
      then
      export PROFILER_COMMAND="-XX:StartFlightRecording=settings=profile,filename=/tmp/${CI_JOB_NAME_SLUG}.jfr,dumponexit=true";
      fi
    - *prepare_test_env
    - export GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xms$GRADLE_MEMORY_MIN -Xmx$GRADLE_MEMORY_MAX $PROFILER_COMMAND -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp -Djava.util.prefs.userRoot=/tmp/.java/.userPrefs-${CI_JOB_ID}' -Ddatadog.forkedMinHeapSize=128M -Ddatadog.forkedMaxHeapSize=1024M"
    - ./gradlew --version
    - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PtestJvm=$testJvm -Pslot=$CI_NODE_INDEX/$CI_NODE_TOTAL $GRADLE_ARGS --continue || $CONTINUE_ON_FAILURE
  after_script:
    - *restore_pretest_env
    - *set_datadog_api_keys
    - *container_info
    - *cgroup_info
    - source .gitlab/gitlab-utils.sh
    - gitlab_section_start "collect-reports" "Collecting reports"
    - .gitlab/collect_reports.sh
    - if [ "$PROFILE_TESTS" == "true" ]; then .gitlab/collect_profiles.sh; fi
    - .gitlab/collect_results.sh
    - .gitlab/upload_ciapp.sh $CACHE_TYPE $testJvm
    - gitlab_section_end "collect-reports"
    - .gitlab/count_tests.sh "$GRADLE_TARGET" "$testJvm" "./results" "./test_counts_${CI_JOB_ID}.json"
    - URL_ENCODED_JOB_NAME=$(jq -rn --arg x "$CI_JOB_NAME" '$x|@uri')
    - echo -e "${TEXT_BOLD}${TEXT_YELLOW}See test results in Datadog:${TEXT_CLEAR} https://app.datadoghq.com/ci/test/runs?query=test_level%3Atest%20%40test.service%3Add-trace-java%20%40ci.pipeline.id%3A${CI_PIPELINE_ID}%20%40ci.job.name%3A%22${URL_ENCODED_JOB_NAME}%22"
  artifacts:
    when: always
    paths:
      - ./reports.tar
      - ./profiles.tar
      - ./results
      - './test_counts_*.json'
      - '.gradle/daemon/*/*.out.log'
    reports:
      junit: results/*.xml
  retry:
    max: 2
    when:
      - unknown_failure
      - stuck_or_timeout_failure
      - runner_system_failure
      - unmet_prerequisites
      - scheduler_failure
      - data_integrity_failure

.test_job_with_test_agent:
  extends: .test_job
  variables:
    CI_USE_TEST_AGENT: "true"
    CI_AGENT_HOST: local-agent
  services:
    - name: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.33.1
      alias: local-agent
      variables:
        LOG_LEVEL: "DEBUG"
        TRACE_LANGUAGE: "java"
        DD_SUPPRESS_TRACE_PARSE_ERRORS: "true"
        DD_POOL_TRACE_CHECK_FAILURES: "true"
        DD_DISABLE_ERROR_RESPONSES: "true"
        ENABLED_CHECKS: "trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service"
  script:
    - !reference [.test_job, script]
    - .gitlab/check_test_agent_results.sh

agent_integration_tests:
  extends: .test_job
  tags: [ "arch:amd64" ]
  variables:
    testJvm: "8"
    CI_AGENT_HOST: local-agent
    GRADLE_TARGET: "traceAgentTest"
    CACHE_TYPE: "base"
  services:
    - name: registry.ddbuild.io/images/mirror/datadog/agent:7.40.1
      alias: local-agent
      variables:
        DD_APM_ENABLED: "true"
        DD_BIND_HOST: "0.0.0.0"
        DD_HOSTNAME: "local-agent"
        DD_API_KEY: "invalid_key_but_this_is_fine"

test_base:
  extends: .test_job
  variables:
    GRADLE_TARGET: ":baseTest"
    CACHE_TYPE: "base"
  parallel:
    matrix: *test_matrix_4
  script:
    - if [ "$testJvm" == "8" ]; then export GRADLE_PARAMS="-PskipFlakyTests -PcheckCoverage"; fi
    - !reference [.test_job, script]

test_inst:
  extends: .test_job_with_test_agent
  variables:
    GRADLE_TARGET: ":instrumentationTest"
    CACHE_TYPE: "inst"
  parallel:
    matrix: *test_matrix_8

test_inst_latest:
  extends: .test_job_with_test_agent
  variables:
    GRADLE_TARGET: ":instrumentationLatestDepTest"
    CACHE_TYPE: "latestdep"
  parallel:
    matrix:
      - testJvm: ["8", "17", "21", "25"] # the latest "stable" version is LTS v25
        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
        # This emulates "parallel" by including it in the matrix
        CI_SPLIT: [ "1/6", "2/6", "3/6", "4/6", "5/6", "6/6"]

test_flaky:
  extends: .test_job_with_test_agent
  variables:
    GRADLE_PARAMS: "-PrunFlakyTests"
    CACHE_TYPE: "smoke"
    testJvm: "8"
    CONTINUE_ON_FAILURE: "true"
  rules:
    - *master_only
    - if: $RUN_FLAKY_TESTS == "true"
      when: on_success
  parallel:
    matrix:
      - GRADLE_TARGET: [":baseTest", ":smokeTest", ":debuggerTest"]
        # Gitlab doesn't support "parallel" and "parallel:matrix" at the same time
        # This emulates "parallel" by including it in the matrix
        CI_SPLIT: [ "1/4", "2/4", "3/4", "4/4" ]

test_flaky_inst:
  extends: .test_job
  variables:
    GRADLE_TARGET: ":instrumentationTest"
    GRADLE_PARAMS: "-PrunFlakyTests"
    CACHE_TYPE: "inst"
    testJvm: "8"
    CONTINUE_ON_FAILURE: "true"
  rules:
    - *master_only
    - if: $RUN_FLAKY_TESTS == "true"
      when: on_success
  parallel: 6

test_profiling:
  extends: .test_job
  variables:
    GRADLE_TARGET: ":profilingTest"
    CACHE_TYPE: "profiling"
  parallel:
    matrix: *test_matrix

# specific jvms list for debugger project because J9-based JVMs have issues with local vars
# so need to test at least against one J9-based JVM
test_debugger:
  extends: .test_job
  variables:
    GRADLE_TARGET: ":debuggerTest"
    CACHE_TYPE: "base"
    DEFAULT_TEST_JVMS: /^(8|11|17|21|25|semeru8)$/ # the latest "stable" version is LTS v25
  parallel:
    matrix: *test_matrix

test_smoke:
  extends: .test_job
  variables:
    GRADLE_TARGET: "stageMainDist :smokeTest"
    GRADLE_PARAMS: "-PskipFlakyTests"
    CACHE_TYPE: "smoke"
  parallel:
    matrix: *test_matrix_8

test_ssi_smoke:
  extends: .test_job
  rules: *master_only
  variables:
    GRADLE_TARGET: "stageMainDist :smokeTest"
    CACHE_TYPE: "smoke"
    DD_INJECT_FORCE: "true"
    DD_INJECTION_ENABLED: "tracer"
  parallel:
    matrix: *test_matrix_8

test_smoke_graalvm:
  extends: .test_job
  tags: [ "arch:amd64" ]
  variables:
    GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test"
    CACHE_TYPE: "smoke"
    CI_NO_SPLIT: "true"
    NON_DEFAULT_JVMS: "true"
  parallel:
    matrix:
      - testJvm: ["graalvm17", "graalvm21", "graalvm25"]

test_smoke_semeru8_debugger:
  extends: .test_job
  tags: [ "arch:amd64" ]
  variables:
    GRADLE_TARGET: "stageMainDist dd-smoke-tests:debugger-integration-tests:test"
    CACHE_TYPE: "smoke"
    NON_DEFAULT_JVMS: "true"
    testJvm: "semeru8"

aggregate_test_counts:
  image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base
  stage: test-summary
  # Note: No explicit 'needs' or 'dependencies' required
  # By default, GitLab CI automatically downloads artifacts from ALL jobs in previous stages
  # This job collects test_counts_*.json files from all test/check jobs via stage ordering
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: always
  script:
    - *set_datadog_api_keys
    - .gitlab/aggregate_test_counts.sh
  artifacts:
    when: always
    paths:
      - test_counts_summary.json
      - test_counts_report.md

deploy_to_profiling_backend:
  stage: publish
  needs: [ build ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: on_success
    - if: '$CI_COMMIT_TAG =~ /^v.*/'
      when: on_success
    - when: manual
      allow_failure: true
  trigger:
    project: DataDog/profiling-backend
    branch: dogfooding
  variables:
    UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME
    UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
    UPSTREAM_TAG: $CI_COMMIT_TAG

trigger_tibco_tests:
  stage: tests
  needs: [ build ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
      when: on_success
    - if: '$CI_COMMIT_TAG =~ /^v.*/'
      when: on_success
    - when: manual
      allow_failure: true
  trigger:
    project: DataDog/tibco-testing
    branch: main
    strategy: depend
  variables:
    UPSTREAM_PACKAGE_JOB: $BUILD_JOB_NAME
    UPSTREAM_PACKAGE_JOB_ID: $BUILD_JOB_ID
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
    UPSTREAM_TAG: $CI_COMMIT_TAG
    FORCE_TRIGGER: $FORCE_TRIGGER

deploy_to_di_backend:manual:
  stage: publish
  needs: [ build ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: manual
      allow_failure: true
  trigger:
    project: DataDog/debugger-demos
    branch: main
  variables:
    UPSTREAM_PACKAGE_JOB: build
    UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
    UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
    UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID
    UPSTREAM_BRANCH: $CI_COMMIT_BRANCH
    UPSTREAM_TAG: $CI_COMMIT_TAG
    UPSTREAM_COMMIT_AUTHOR: $CI_COMMIT_AUTHOR
    UPSTREAM_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA

# If the deploy_to_maven_central job is re-run, re-trigger the deploy_artifacts_to_github job as well so that the artifacts match.
deploy_to_maven_central:
  extends: .gradle_build
  stage: publish
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_BRANCH == "master"'
      when: on_success
    # Do not deploy release candidate versions
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
    - when: manual
      allow_failure: true
  script:
    - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
    - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID publishToSonatype closeSonatypeStagingRepository -PskipTests $GRADLE_ARGS
  artifacts:
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'

deploy_snapshot_with_ddprof_snapshot:
  extends: .gradle_build
  stage: publish
  needs: [ build ]
  variables:
    CACHE_TYPE: "lib"
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    # Manual trigger only - for testing with ddprof snapshot versions
    - when: manual
      allow_failure: true
  script:
    - export MAVEN_CENTRAL_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_username --with-decryption --query "Parameter.Value" --out text)
    - export MAVEN_CENTRAL_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.central_password --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
    - export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-java.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
    - echo "Publishing dd-trace-java snapshot with ddprof snapshot dependency"
    - ./gradlew -PbuildInfo.build.number=$CI_JOB_ID -PddprofUseSnapshot publishToSonatype -PskipTests $GRADLE_ARGS
  artifacts:
    paths:
      - 'workspace/dd-java-agent/build/libs/*.jar'
      - 'workspace/dd-trace-api/build/libs/*.jar'
      - 'workspace/dd-trace-ot/build/libs/*.jar'

deploy_artifacts_to_github:
  stage: publish
  image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
  tags: [ "arch:amd64" ]
  id_tokens:
    DDOCTOSTS_ID_TOKEN:
      aud: dd-octo-sts
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
  # Requires the deploy_to_maven_central job to have run first (the UP-TO-DATE gradle check across jobs is broken)
  # This will deploy the artifacts built from the publishToSonatype task to the GitHub release
  needs:
    - job: deploy_to_maven_central
      # The deploy_to_maven_central job is not run for release candidate versions
      optional: true
  before_script:
    - dd-octo-sts version
    - dd-octo-sts debug --scope DataDog/dd-trace-java --policy self.gitlab.release
    - dd-octo-sts token --scope DataDog/dd-trace-java --policy self.gitlab.release > github-token.txt
  script:
    - gh auth login --with-token < github-token.txt
    - gh auth status
    - export VERSION=${CI_COMMIT_TAG##v} # remove "v" from front of tag to get version
    - cp workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar workspace/dd-java-agent/build/libs/dd-java-agent.jar # upload two filenames
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-java-agent/build/libs/dd-java-agent-${VERSION}.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-api/build/libs/dd-trace-api-${VERSION}.jar
    - gh release upload --clobber --repo DataDog/dd-trace-java $CI_COMMIT_TAG workspace/dd-trace-ot/build/libs/dd-trace-ot-${VERSION}.jar
  after_script:
    - dd-octo-sts revoke -t $(cat github-token.txt)
  retry:
    max: 2
    when: always

requirements_json_test:
  rules:
    - when: on_success
  variables:
    REQUIREMENTS_BLOCK_JSON_PATH: "metadata/requirements-block.json"
    REQUIREMENTS_ALLOW_JSON_PATH: "metadata/requirements-allow.json"

package-oci:
  needs: [ build ]

override_verify_maven_central:
  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release
  stage: publish
  needs: [ ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - when: manual
      allow_failure: true
  script:
    - touch OVERRIDE_MAVEN_VERIFY
  cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs
    - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL
      paths:
        - OVERRIDE_MAVEN_VERIFY
      policy: push
      unprotect: true

# Verify Maven Central deployment is publicly available before publishing OCI images
verify_maven_central_deployment:
  image: registry.ddbuild.io/images/base/gbi-ubuntu_2204:release
  stage: publish
  needs: [ deploy_to_maven_central ]
  rules:
    - if: '$POPULATE_CACHE'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+\.[0-9]+$/'
      when: on_success
  cache: # Cache is used to signal between the override_verify_maven_central and verify_maven_central_deployment jobs
    - key: $CI_PIPELINE_ID-OVERRIDE_SIGNAL
      paths:
        - OVERRIDE_MAVEN_VERIFY
      policy: pull
      unprotect: true
  script:
    - if [ -f OVERRIDE_MAVEN_VERIFY ]; then echo "SKIPPING MAVEN VERIFICATION"; exit 0; fi
    - |
      export VERSION=${CI_COMMIT_TAG##v}
      ARTIFACT_URLS=(
        "https://repo1.maven.org/maven2/com/datadoghq/dd-java-agent/${VERSION}/dd-java-agent-${VERSION}.jar"
        "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-api/${VERSION}/dd-trace-api-${VERSION}.jar"
        "https://repo1.maven.org/maven2/com/datadoghq/dd-trace-ot/${VERSION}/dd-trace-ot-${VERSION}.jar"
      )
      # Wait 5 mins initially, then try 5 times with a minute delay between each retry to see if the release artifacts are available
      sleep 300
      TRY=0
      MAX_TRIES=5
      DELAY=60
      while [ $TRY -lt $MAX_TRIES ]; do
        ARTIFACTS_AVAILABLE=true
        for URL in "${ARTIFACT_URLS[@]}"; do
          if ! curl --location --fail --silent --show-error -I "$URL"; then
            ARTIFACTS_AVAILABLE=false
            break
          fi
        done
        if [ "$ARTIFACTS_AVAILABLE" = true ]; then
          break
        fi
        TRY=$((TRY + 1))
        if [ $TRY -eq $MAX_TRIES ]; then
          echo "The release was not available after 10 mins. Manually re-run the job to try again."
          exit 1
        fi
        sleep $DELAY
      done

publishing-gate:
  stage: publish
  needs:
    - job: verify_maven_central_deployment
      optional: true # Required for releases only

configure_system_tests:
  variables:
    SYSTEM_TESTS_SCENARIOS_GROUPS: "simple_onboarding,simple_onboarding_profiling,simple_onboarding_appsec,docker-ssi,lib-injection"

create_key:
  stage: generate-signing-key
  when: manual
  needs: [ ]
  variables:
    PROJECT_NAME: "dd-trace-java"
    EXPORT_TO_KEYSERVER: "true"
  image: $REGISTRY/ci/agent-key-management-tools/gpg:1
  script:
    - /create.sh
  artifacts:
    expire_in: 13 mos
    paths:
      - pubkeys

validate_supported_configurations_v2_local_file:
  extends: .validate_supported_configurations_v2_local_file
  variables:
    LOCAL_JSON_PATH: "metadata/supported-configurations.json"
    BACKFILLED: "false"

update_central_configurations_version_range_v2:
  extends: .update_central_configurations_version_range_v2
  variables:
    LOCAL_REPO_NAME: "dd-trace-java"
    LOCAL_JSON_PATH: "metadata/supported-configurations.json"
    LANGUAGE_NAME: "java"
    MULTIPLE_RELEASE_LINES: "false"