Use a dynamic environment for PR jobs

Kobzol · Kobzol · commit 65141abc8f44 · 2024-04-19T23:17:33.000+02:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -45,23 +45,23 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          GITHUB_CTX: "${{ toJSON(github) }}"
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
-  pr:
-    name: "PR - ${{ matrix.name }}"
+  job:
+    name: "${{ matrix.name }}"
     needs:
       - calculate_matrix
     env:
-      PR_CI_JOB: 1
-      CI_JOB_NAME: "${{ matrix.name }}"
+      CI_JOB_NAME: "${{ matrix.image }}"
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
       DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
       CACHE_DOMAIN: ci-caches.rust-lang.org
-    if: "github.event_name == 'pull_request'"
-    continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
+    continue-on-error: "${{ matrix.continue_on_error || false }}"
     strategy:
       matrix:
         include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
@@ -589,145 +589,6 @@ jobs:
           AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
           AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
         if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
-  try:
-    name: "try - ${{ matrix.name }}"
-    env:
-      DIST_TRY_BUILD: 1
-      CI_JOB_NAME: "${{ matrix.name }}"
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
-      DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      DEPLOY_BUCKET: rust-lang-ci2
-      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
-      TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
-      TOOLSTATE_PUBLISH: 1
-      CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
-      ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
-      AWS_REGION: us-west-1
-      CACHE_DOMAIN: ci-caches.rust-lang.org
-    if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))"
-    strategy:
-      matrix:
-        include:
-          - name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: "llvm,cranelift"
-            os: ubuntu-20.04-16core-64gb
-    defaults:
-      run:
-        shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
-    timeout-minutes: 600
-    runs-on: "${{ matrix.os }}"
-    steps:
-      - if: "contains(matrix.os, 'windows')"
-        uses: msys2/setup-msys2@v2.22.0
-        with:
-          msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
-          update: false
-          release: true
-          path-type: inherit
-          install: "make dos2unix diffutils\n"
-      - name: disable git crlf conversion
-        run: git config --global core.autocrlf false
-      - name: checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-      - name: configure the PR in which the error message will be posted
-        run: "echo \"[CI_PR_NUMBER=$num]\""
-        env:
-          num: "${{ github.event.number }}"
-        if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
-      - name: add extra environment variables
-        run: src/ci/scripts/setup-environment.sh
-        env:
-          EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
-        if: success() && !env.SKIP_JOB
-      - name: decide whether to skip this job
-        run: src/ci/scripts/should-skip-this.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure the channel matches the target branch
-        run: src/ci/scripts/verify-channel.sh
-        if: success() && !env.SKIP_JOB
-      - name: collect CPU statistics
-        run: src/ci/scripts/collect-cpu-stats.sh
-        if: success() && !env.SKIP_JOB
-      - name: show the current environment
-        run: src/ci/scripts/dump-environment.sh
-        if: success() && !env.SKIP_JOB
-      - name: install awscli
-        run: src/ci/scripts/install-awscli.sh
-        if: success() && !env.SKIP_JOB
-      - name: install sccache
-        run: src/ci/scripts/install-sccache.sh
-        if: success() && !env.SKIP_JOB
-      - name: select Xcode
-        run: src/ci/scripts/select-xcode.sh
-        if: success() && !env.SKIP_JOB
-      - name: install clang
-        run: src/ci/scripts/install-clang.sh
-        if: success() && !env.SKIP_JOB
-      - name: install tidy
-        run: src/ci/scripts/install-tidy.sh
-        if: success() && !env.SKIP_JOB
-      - name: install WIX
-        run: src/ci/scripts/install-wix.sh
-        if: success() && !env.SKIP_JOB
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
-      - name: checkout submodules
-        run: src/ci/scripts/checkout-submodules.sh
-        if: success() && !env.SKIP_JOB
-      - name: install MSYS2
-        run: src/ci/scripts/install-msys2.sh
-        if: success() && !env.SKIP_JOB
-      - name: install MinGW
-        run: src/ci/scripts/install-mingw.sh
-        if: success() && !env.SKIP_JOB
-      - name: install ninja
-        run: src/ci/scripts/install-ninja.sh
-        if: success() && !env.SKIP_JOB
-      - name: enable ipv6 on Docker
-        run: src/ci/scripts/enable-docker-ipv6.sh
-        if: success() && !env.SKIP_JOB
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure line endings are correct
-        run: src/ci/scripts/verify-line-endings.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure backported commits are in upstream branches
-        run: src/ci/scripts/verify-backported-commits.sh
-        if: success() && !env.SKIP_JOB
-      - name: ensure the stable version number is correct
-        run: src/ci/scripts/verify-stable-version-number.sh
-        if: success() && !env.SKIP_JOB
-      - name: run the build
-        run: src/ci/scripts/run-build-from-ci.sh 2>&1
-        env:
-          AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
-          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
-        if: success() && !env.SKIP_JOB
-      - name: create github artifacts
-        run: src/ci/scripts/create-doc-artifacts.sh
-        if: success() && !env.SKIP_JOB
-      - name: upload artifacts to github
-        uses: actions/upload-artifact@v4
-        with:
-          name: "${{ env.DOC_ARTIFACT_NAME }}"
-          path: obj/artifacts/doc
-          if-no-files-found: ignore
-          retention-days: 5
-        if: success() && !env.SKIP_JOB
-      - name: upload artifacts to S3
-        run: src/ci/scripts/upload-artifacts.sh
-        env:
-          AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
-          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
-        if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
   master:
     name: master
     runs-on: ubuntu-latest
diff --git a/library/backtrace b/library/backtrace
@@ -1 +1 @@
-Subproject commit e15130618237eb3e2d4b622549f9647b4c1d9ca3
+Subproject commit 6fa4b85b9962c3e1be8c2e5cc605cd078134152b
diff --git a/library/stdarch b/library/stdarch
@@ -1 +1 @@
-Subproject commit 7df81ba8c3e2d02c2ace0c5a6f4f32d800c09e56
+Subproject commit 967e7afd87cbea3232581a4a55031134ab88f595
diff --git a/src/ci/github-actions/calculate-job-matrix.py b/src/ci/github-actions/calculate-job-matrix.py
@@ -11,15 +11,41 @@
 """
 
 import json
+import os
 from pathlib import Path
+from typing import List, Dict
 
 import yaml
 
 JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
 
 
+def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
+    for job in jobs:
+        job["name"] = f"{prefix} - {job['image']}"
+    return jobs
+
+
 if __name__ == "__main__":
+    github_ctx = json.loads(os.environ["GITHUB_CTX"])
+
     with open(JOBS_YAML_PATH) as f:
-        jobs = yaml.safe_load(f)
-    job_output = jobs["pr"]
-    print(f"jobs={json.dumps(job_output)}")
+        data = yaml.safe_load(f)
+
+    event_name = github_ctx["event_name"]
+    ref = github_ctx["ref"]
+    repository = github_ctx["repository"]
+
+    old_bors_try_build = ref in ("refs/heads/try", "refs/heads/try-perf") and repository == "rust-lang-ci/rust"
+    new_bors_try_build = ref == "refs/heads/automation/bors/try" and repository == "rust-lang/rust"
+    try_build = old_bors_try_build or new_bors_try_build
+
+    jobs = []
+    # Pull request CI jobs. Their name is 'PR - <image>'
+    if event_name == "pull_request":
+        jobs = name_jobs(data["pr"], "PR")
+    # Try builds
+    elif event_name == "push" and try_build:
+        jobs = name_jobs(data["try"], "try")
+
+    print(f"jobs={json.dumps(jobs)}")
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
@@ -351,17 +351,24 @@ jobs:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
-        run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
+        env:
+          GITHUB_CTX: ${{ toJSON(github) }}
+        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
-  pr:
+  job:
     <<: *base-ci-job
-    name: PR - ${{ matrix.name }}
+    name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
     env:
-      <<: [*shared-ci-variables, *public-variables]
-      PR_CI_JOB: 1
-    if: github.event_name == 'pull_request'
-    continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
+      CI_JOB_NAME: ${{ matrix.image }}
+      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SCCACHE_BUCKET: rust-lang-ci-sccache2
+      TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+      CACHE_DOMAIN: ci-caches.rust-lang.org
+    continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
         # Check the `calculate_matrix` job to see how is the matrix defined.
@@ -751,23 +758,6 @@ jobs:
               SCRIPT: python x.py dist bootstrap --include-default-paths
             <<: *job-windows-8c
 
-  try:
-    <<: *base-ci-job
-    name: try - ${{ matrix.name }}
-    env:
-      DIST_TRY_BUILD: 1
-      <<: [*shared-ci-variables, *prod-variables]
-    if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
-    strategy:
-      matrix:
-        include:
-          - &dist-x86_64-linux
-            name: dist-x86_64-linux
-            env:
-              CODEGEN_BACKENDS: llvm,cranelift
-            <<: *job-linux-16c
-
-
   master:
     name: master
     runs-on: ubuntu-latest
diff --git a/src/ci/github-actions/jobs.yml b/src/ci/github-actions/jobs.yml
@@ -2,7 +2,7 @@
 # dynamically in CI from ci.yml.
 # You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
 # modify this file.
-shared_defs:
+runners:
   - &base-job
     env: { }
 
@@ -37,14 +37,52 @@ shared_defs:
   - &job-aarch64-linux
     os: [ self-hosted, ARM64, linux ]
 
+envs:
+  - &pr-env
+    PR_CI_JOB: 1
+
+  - &prod-variables
+    SCCACHE_BUCKET: rust-lang-ci-sccache2
+    DEPLOY_BUCKET: rust-lang-ci2
+    TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
+    TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
+    TOOLSTATE_PUBLISH: 1
+    # AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
+    # AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
+    # rotate them in a single branch while keeping the old key in another
+    # branch, which wouldn't be possible if the key was named with the kind
+    # (caches, artifacts...).
+    CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
+    ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
+    AWS_REGION: us-west-1
+    CACHE_DOMAIN: ci-caches.rust-lang.org
+
+# Pull request jobs
 pr:
-  - name: mingw-check
+  - image: mingw-check
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: mingw-check-tidy
+  - image: mingw-check-tidy
+    continue_on_error: true
+    env:
+      <<: *pr-env
     <<: *job-linux-4c
-  - name: x86_64-gnu-llvm-17
+  - image: x86_64-gnu-llvm-17
     env:
       ENABLE_GCC_CODEGEN: "1"
+      <<: *pr-env
     <<: *job-linux-16c
-  - name: x86_64-gnu-tools
+  - image: x86_64-gnu-tools
+    env:
+      <<: *pr-env
+    <<: *job-linux-16c
+
+# Try build jobs
+try:
+  - image: dist-x86_64-linux
+    env:
+      DIST_TRY_BUILD: 1
+      CODEGEN_BACKENDS: llvm,cranelift
+      <<: *prod-variables
     <<: *job-linux-16c
diff --git a/src/doc/book b/src/doc/book
@@ -1 +1 @@
-Subproject commit 3131aa4642c627a24f523c82566b94a7d920f68c
+Subproject commit 19c40bfd2d57641d962f3119a1c343355f1b3c5e
diff --git a/src/doc/edition-guide b/src/doc/edition-guide
@@ -1 +1 @@
-Subproject commit eb3eb80e106d03250c1fb7c5666b1c8c59672862
+Subproject commit 98b33e9a441457b0a491fe1be90e7de64eafc3e5
diff --git a/src/doc/embedded-book b/src/doc/embedded-book
@@ -1 +1 @@
-Subproject commit aa7d4b0b4653ddb47cb1de2036d090ec2ba9dab1
+Subproject commit 2e95fc2fd31d669947e993aa07ef10dc9828bee7
diff --git a/src/doc/nomicon b/src/doc/nomicon
@@ -1 +1 @@
-Subproject commit 0d5f88475fe285affa6dbbc806e9e44d730797c0
+Subproject commit 6bc2415218d4dd0cb01433d8320f5ccf79c343a1
diff --git a/src/doc/reference b/src/doc/reference
@@ -1 +1 @@
-Subproject commit 55694913b1301cc809f9bf4a1ad1b3d6920efbd9
+Subproject commit 984b36eca4b9293df04d5ba4eb5c4f77db0f51dc
diff --git a/src/doc/rust-by-example b/src/doc/rust-by-example
@@ -1 +1 @@
-Subproject commit 60d34b5fd33db1346f9aabfc0c9d0bda6c8e42be
+Subproject commit 7601e0c5ad29d5bd3b518700ea63fddfff5915a7
diff --git a/src/doc/rustc-dev-guide b/src/doc/rustc-dev-guide
@@ -1 +1 @@
-Subproject commit b77a34bd46399687b4ce6a17198e9f316c988794
+Subproject commit ffa246b7fd95a96e1cd54883e613aed42c32547d
diff --git a/src/llvm-project b/src/llvm-project
@@ -1 +1 @@
-Subproject commit 5399a24c66cb6164cf32280e7d300488c90d5765
+Subproject commit 0af6c732ec6ca189cd7725e4a7d4290793046e83
diff --git a/src/tools/cargo b/src/tools/cargo
@@ -1 +1 @@
-Subproject commit 6f06fe908a5ee0f415c187f868ea627e82efe07d
+Subproject commit 0637083df5bbdcc951845f0d2eff6999cdb6d30a
