CI: updating the main container flow

CI: extend with github <> gitlab link

CI: hotfix followup

CI: hotfixes

CI: hotfix for job

CI: patch test scripts

CI: configure GH token permission

CI: configure gitlab test

CI: switch to PAT token (CI token expires once job is done)

CI: finalize test script

Author: DeviaVir

.github/workflows/start-gitlab.yml

@@ -0,0 +1,53 @@
+name: Start GitLab CI
+on:
+  # Use pull_request_target to run the workflow from the base branch (e.g., main)
+  # This ensures the trusted workflow logic executes, even for PRs from forks.
+  # It also grants access to secrets needed for the trigger.
+  pull_request_target:
+    types: [opened, synchronize, reopened]
+jobs:
+  trigger-gitlab:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Start Gitlab Pipeline
+        env:
+          # Get trigger config from secrets
+          GL_TRIGGER_TOKEN: ${{ secrets.GL_TRIGGER_TOKEN }}
+          GL_TRIGGER_URL: ${{ secrets.GL_TRIGGER_URL }}
+          # Use a specific ref from secrets if provided, otherwise default to the PR's head branch name
+          GL_TRIGGER_REF: ${{ secrets.GL_TRIGGER_REF || github.event.pull_request.head.ref }}
+          # --- Variables to pass to GitLab ---
+          # The commit SHA in the GitHub PR
+          GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha }}
+          # The ref (branch name) of the PR head
+          GITHUB_PR_REF: ${{ github.event.pull_request.head.ref }}
+          # The repository name (e.g., 'your-org/your-repo')
+          GITHUB_REPO: ${{ github.repository }}
+          # The GitHub token for reporting status back
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # --- Safety Checks ---
+          # Ensure critical secrets are actually available (they should be with pull_request_target)
+          if [ -z "$GL_TRIGGER_TOKEN" ]; then
+            echo "::error::GL_TRIGGER_TOKEN secret is missing or unavailable!"
+            exit 1
+          fi
+          if [ -z "$GITHUB_TOKEN" ]; then
+            echo "::error::GITHUB_TOKEN is empty. Secrets may not be properly accessed."
+            exit 1
+          fi
+          # Ensure URL is set
+          if [ -z "$GL_TRIGGER_URL" ]; then
+            echo "::error::GL_TRIGGER_URL secret is missing or unavailable!"
+            exit 1
+          fi
+
+          echo "Triggering GitLab pipeline for SHA: ${GITHUB_PR_SHA}"
+          curl --fail --silent --show-error --request POST \
+            --form token="${GL_TRIGGER_TOKEN}" \
+            --form ref="${GL_TRIGGER_REF}" \
+            --form "variables[GITHUB_PR_SHA]=${GITHUB_PR_SHA}" \
+            --form "variables[GITHUB_PR_REF]=${GITHUB_PR_REF}" \
+            --form "variables[GITHUB_REPO]=${GITHUB_REPO}" \
+            "${GL_TRIGGER_URL}" > /dev/null
+          echo "GitLab pipeline triggered."

.gitlab-ci.yml

@@ -3,7 +3,6 @@ variables:
   DOCKER_TLS_CERTDIR: ""
   DOCKER_BUILDKIT: 1
   CI_DISPOSABLE_ENVIRONMENT: "true"
-  BULLSEYE_IMAGE: debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
   IMAGE_BASE: blockstream/esplora-base
   IMAGE: blockstream/esplora
   DOCKERHUB_ESPLORA_URL: "https://hub.docker.com/v2/repositories/blockstream/esplora/tags/"
@@ -12,15 +11,8 @@ default:
   image: docker:27
   services:
     - name: docker:27-dind
-      command: ["dockerd", "--host=tcp://0.0.0.0:2375"]
+      command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--mtu=1450"]
       alias: "docker"
-  before_script:
-    - docker info
-    - docker buildx create
-      --driver=docker-container
-      --name=buildkit-builder
-      --use
-      --platform linux/amd64,linux/arm64
   tags:
     - cloud
   retry:

gitlab/build.yml

@@ -1,44 +1,132 @@
-build_base:
+.build_esplora_base:
   stage: build
   rules:
     - if: $CI_COMMIT_BRANCH
       changes:
         paths:
           - Dockerfile.deps
         compare_to: master
+  before_script:
+    - docker buildx create
+      --driver=docker-container
+      --name=buildkit-builder
+      --use
+      --platform linux/${ARCH}
   script:
+    - docker pull ${IMAGE_BASE}:latest-${ARCH} || true
     - docker buildx build
-      --platform linux/amd64,linux/arm64
       --push
+      --platform linux/${ARCH}
       -f Dockerfile.deps
       --build-arg BUILDKIT_INLINE_CACHE=1
-      --cache-from ${IMAGE_BASE}:latest
-      -t ${IMAGE_BASE}:$CI_COMMIT_SHA
+      --cache-from ${IMAGE_BASE}:latest-${ARCH}
+      -t ${IMAGE_BASE}:latest-${ARCH} .
+build_esplora_base_amd64:
+  extends:
+    - .build_esplora_base
+  variables:
+    ARCH: amd64
+build_esplora_base_arm64:
+  extends:
+    - .build_esplora_base
+  variables:
+    ARCH: arm64
+  tags:
+    - cloud-arm
+build_and_push_esplora_base:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        paths:
+          - Dockerfile.deps
+        compare_to: master
+  needs:
+    - build_esplora_base_amd64
+    - build_esplora_base_arm64
+  script:
+    - docker buildx imagetools create
       -t ${IMAGE_BASE}:latest
-      .
+      ${IMAGE_BASE}:latest-amd64
+      ${IMAGE_BASE}:latest-arm64
 
-test_docker_build_esplora:
+.build_esplora_test:
   stage: build
+  only:
+    - merge_requests
+  before_script:
+    - docker buildx create
+      --driver=docker-container
+      --name=buildkit-builder
+      --use
+      --platform linux/${ARCH}
   script:
+    - docker pull ${IMAGE}:latest-${ARCH} || true
     - docker buildx build
-      --platform linux/amd64,linux/arm64
+      --platform linux/${ARCH}
       --build-arg BUILDKIT_INLINE_CACHE=1
-      --cache-from ${IMAGE}:latest
+      --cache-from ${IMAGE}:latest-${ARCH}
       -f Dockerfile
       -t ${IMAGE} .
+build_esplora_test_amd64:
+  extends:
+    - .build_esplora_test
+  variables:
+    ARCH: amd64
+build_esplora_test_arm64:
+  tags:
+    - cloud-arm
+  extends:
+    - .build_esplora_test
+  variables:
+    ARCH: arm64
 
-build_esplora:
+.build_esplora_latest:
   stage: build
-  when: manual
+  only:
+    - master
+  except:
+    - triggers
+  before_script:
+    - docker buildx create
+      --driver=docker-container
+      --name=buildkit-builder
+      --use
+      --platform linux/${ARCH}
   script:
+    - docker pull ${IMAGE}:latest-${ARCH} || true
     - curl -s "${DOCKERHUB_ESPLORA_URL}" | grep -q "$CI_COMMIT_SHA" || (
       sed -i "s#esplora-base:latest#esplora-base:${BASE_TAG}#" Dockerfile
       && docker buildx build
-      --platform linux/amd64,linux/arm64
+      --platform linux/${ARCH}
       --push
       --build-arg BUILDKIT_INLINE_CACHE=1
       --build-arg FOOT_HTML='<!-- '"$CI_COMMIT_SHA"' -->'
-      --cache-from ${IMAGE}:latest
+      --cache-from ${IMAGE}:latest-${ARCH}
+      -t ${IMAGE}:latest-${ARCH} .)
+build_esplora_latest_amd64:
+  extends:
+    - .build_esplora_latest
+  variables:
+    ARCH: amd64
+build_esplora_latest_arm64:
+  tags:
+    - cloud-arm
+  extends:
+    - .build_esplora_latest
+  variables:
+    ARCH: arm64
+build_and_push_esplora_latest:
+  stage: build
+  only:
+    - master
+  except:
+    - triggers
+  needs:
+    - build_esplora_latest_amd64
+    - build_esplora_latest_arm64
+  script:
+    - docker buildx imagetools create
       -t ${IMAGE}:latest
-      -t ${IMAGE}:$CI_COMMIT_SHA .)
-    - if [ $CI_COMMIT_BRANCH == "master" ]; then docker pull ${IMAGE}:$CI_COMMIT_SHA; docker tag ${IMAGE}:$CI_COMMIT_SHA ${IMAGE}:latest; docker push ${IMAGE}:latest; fi
+      ${IMAGE}:latest-amd64
+      ${IMAGE}:latest-arm64

gitlab/test.yml

@@ -0,0 +1,73 @@
+report_github_status:
+  stage: build
+  variables:
+    GIT_STRATEGY: none
+    ARCH: amd64
+  rules:
+    # Only run this job if:
+    # 1. The pipeline was started by an API trigger (`CI_PIPELINE_SOURCE == "trigger"`)
+    # 2. The required variables from GitHub Actions are present.
+    - if: '$CI_PIPELINE_SOURCE == "trigger" && $GITHUB_PR_SHA && $GH_STATUS_TOKEN && $GITHUB_REPO && $GITHUB_PR_REF'
+  before_script:
+    # Make sure curl and git are available
+    - apk add --no-cache curl git
+    # Report "pending" status to GitHub as soon as the job starts
+    - |
+      echo "Reporting pending status to GitHub commit $GITHUB_PR_SHA"
+      curl --fail --request POST \
+        --url "https://api.github.com/repos/${GITHUB_REPO}/statuses/${GITHUB_PR_SHA}" \
+        --header "Authorization: Bearer ${GH_STATUS_TOKEN}" \
+        --header "Accept: application/vnd.github.v3+json" \
+        --header "Content-Type: application/json" \
+        --data @- <<EOF
+      {
+        "state": "pending",
+        "target_url": "${CI_PIPELINE_URL}",
+        "description": "GitLab CI pipeline is running...",
+        "context": "ci/gitlab/pipeline-status"
+      }
+      EOF
+    - |
+      docker buildx create \
+      --driver=docker-container \
+      --name=buildkit-builder \
+      --use \
+      --platform linux/${ARCH}
+  script:
+    - git clone -b "$GITHUB_PR_REF" https://github.com/blockstream/esplora.git .
+    - docker pull "${IMAGE_BASE}:latest-${ARCH}" || true
+    - |
+      docker buildx build \
+        --push \
+        --platform "linux/${ARCH}" \
+        -f Dockerfile.deps \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        --cache-from "${IMAGE_BASE}:latest-${ARCH}" \
+        -t "${IMAGE_BASE}:latest-${ARCH}" .
+  after_script:
+    # Report final status ("success" or "failure") to GitHub after the job finishes
+    # This block runs even if the main 'script' fails
+    - |
+      FINAL_STATE="success" # Assume success
+      DESCRIPTION="GitLab CI pipeline succeeded."
+      # Check the GitLab CI job status variable
+      if [ -z "${CI_JOB_STATUS}" ] || [ "${CI_JOB_STATUS}" != "success" ]; then
+        echo "Job status was $CI_JOB_STATUS, reporting failure."
+        FINAL_STATE="failure"
+        DESCRIPTION="GitLab CI pipeline failed."
+      fi
+
+      echo "Reporting $FINAL_STATE status to GitHub commit $GITHUB_PR_SHA"
+      curl --fail --request POST \
+        --url "https://api.github.com/repos/${GITHUB_REPO}/statuses/${GITHUB_PR_SHA}" \
+        --header "Authorization: Bearer ${GH_STATUS_TOKEN}" \
+        --header "Accept: application/vnd.github.v3+json" \
+        --header "Content-Type: application/json" \
+        --data @- << EOF
+      {
+        "state": "${FINAL_STATE}",
+        "target_url": "${CI_PIPELINE_URL}",
+        "description": "${DESCRIPTION}",
+        "context": "ci/gitlab/pipeline-status" # Use the same context as 'pending'
+      }
+      EOF