1.85.0: Add CI files and use rzup to build toolchain (#21)

* add CI files

* use latest rzup

* run ci

* fix path

* fix

Author: Erik Kaneda

.github/workflows/ci.yml

@@ -1,264 +1,56 @@
-# This file defines our primary CI workflow that runs on pull requests
-# and also on pushes to special branches (auto, try).
-#
-# The actual definition of the executed jobs is calculated by a Python
-# script located at src/ci/github-actions/calculate-job-matrix.py, which
-# uses job definition data from src/ci/github-actions/jobs.yml.
-# You should primarily modify the `jobs.yml` file if you want to modify
-# what jobs are executed in CI.
-
 name: CI
+
 on:
   push:
-    branches:
-      - auto
-      - try
-      - try-perf
-      - automation/bors/try
+    branches: [ risc0 ]
   pull_request:
-    branches:
-      - "**"
-
-permissions:
-  contents: read
-  packages: write
+    branches: [ risc0, "risc0-*" ]
+  workflow_call:
+  workflow_dispatch:
 
-defaults:
-  run:
-    # On Linux, macOS, and Windows, use the system-provided bash as the default
-    # shell. (This should only make a difference on Windows, where the default
-    # shell is PowerShell.)
-    shell: bash
-
-concurrency:
-  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
-  # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
-  # are all triggered on the same branch, but which should be able to run concurrently.
-  group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
-  cancel-in-progress: true
-env:
-  TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
-  # This will be empty in PR jobs.
-  TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
 jobs:
-  # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
-  # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
-  # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
-  calculate_matrix:
-    name: Calculate job matrix
-    runs-on: ubuntu-24.04
-    outputs:
-      jobs: ${{ steps.jobs.outputs.jobs }}
-      run_type: ${{ steps.jobs.outputs.run_type }}
-    steps:
-      - name: Checkout the source code
-        uses: actions/checkout@v4
-      - name: Calculate the CI job matrix
-        env:
-          COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
-        run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
-        id: jobs
-  job:
-    name: ${{ matrix.name }}
-    needs: [ calculate_matrix ]
-    runs-on: "${{ matrix.os }}"
-    defaults:
-      run:
-        shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
-    timeout-minutes: 360
-    env:
-      CI_JOB_NAME: ${{ matrix.image }}
-      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
-      # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
-      HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
-      DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      SCCACHE_BUCKET: rust-lang-ci-sccache2
-      CACHE_DOMAIN: ci-caches.rust-lang.org
-    continue-on-error: ${{ matrix.continue_on_error || false }}
+  build:
     strategy:
+      fail-fast: false
       matrix:
-        # Check the `calculate_matrix` job to see how is the matrix defined.
-        include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
+        include:
+          - os: macOS
+            arch: ARM64
+            triple: aarch64-apple-darwin
+          - os: Linux
+            arch: X64
+            triple: x86_64-unknown-linux-gnu
+    runs-on: [ self-hosted, prod, "${{ matrix.os }}", "${{ matrix.arch }}" ]
     steps:
-      - if: contains(matrix.os, 'windows')
-        uses: msys2/[email protected]
+      - name: Install Rust
+        uses: risc0/actions-rs-toolchain@v1
         with:
-          # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
-          msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
-          # don't try to download updates for already installed packages
-          update: false
-          # don't try to use the msys that comes built-in to the github runner,
-          # so we can control what is installed (i.e. not python)
-          release: true
-          # Inherit the full path from the Windows environment, with MSYS2's */bin/
-          # dirs placed in front. This lets us run Windows-native Python etc.
-          path-type: inherit
-          install: >
-            make
+            toolchain: stable
+      - uses: lukka/[email protected]
 
-      - name: disable git crlf conversion
-        run: git config --global core.autocrlf false
+      - name: Check out risc0/rust
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+          path: rust
+          fetch-depth: 0
 
-      - name: checkout the source code
+      - name: Check out risc0/risc0
         uses: actions/checkout@v4
         with:
-          fetch-depth: 2
-
-      # Free up disk space on Linux by removing preinstalled components that
-      # we do not need. We do this to enable some of the less resource
-      # intensive jobs to run on free runners, which however also have
-      # less disk space.
-      - name: free up disk space
-        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
-        if: matrix.free_disk
-
-      # Rust Log Analyzer can't currently detect the PR number of a GitHub
-      # Actions build on its own, so a hint in the log message is needed to
-      # point it in the right direction.
-      - name: configure the PR in which the error message will be posted
-        run: echo "[CI_PR_NUMBER=$num]"
-        env:
-          num: ${{ github.event.number }}
-        if: needs.calculate_matrix.outputs.run_type == 'pr'
-
-      - name: add extra environment variables
-        run: src/ci/scripts/setup-environment.sh
-        env:
-          # Since it's not possible to merge `${{ matrix.env }}` with the other
-          # variables in `job.<name>.env`, the variables defined in the matrix
-          # are passed to the `setup-environment.sh` script encoded in JSON,
-          # which then uses log commands to actually set them.
-          EXTRA_VARIABLES: ${{ toJson(matrix.env) }}
-
-      - name: ensure the channel matches the target branch
-        run: src/ci/scripts/verify-channel.sh
-
-      - name: collect CPU statistics
-        run: src/ci/scripts/collect-cpu-stats.sh
-
-      - name: show the current environment
-        run: src/ci/scripts/dump-environment.sh
-
-      - name: install awscli
-        run: src/ci/scripts/install-awscli.sh
-
-      - name: install sccache
-        run: src/ci/scripts/install-sccache.sh
-
-      - name: select Xcode
-        run: src/ci/scripts/select-xcode.sh
-
-      - name: install clang
-        run: src/ci/scripts/install-clang.sh
-
-      - name: install tidy
-        run: src/ci/scripts/install-tidy.sh
+          repository: risc0/risc0
+          ref: main
+          path: risc0
 
-      - name: install WIX
-        run: src/ci/scripts/install-wix.sh
+      - name: Build
+        run: GITHUB_ACTIONS=false cargo run --bin rzup -- build rust --path $GITHUB_WORKSPACE/rust
+        working-directory: risc0
 
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-
-      - name: checkout submodules
-        run: src/ci/scripts/checkout-submodules.sh
-
-      - name: install MinGW
-        run: src/ci/scripts/install-mingw.sh
-
-      - name: install ninja
-        run: src/ci/scripts/install-ninja.sh
-
-      - name: enable ipv6 on Docker
-        run: src/ci/scripts/enable-docker-ipv6.sh
-
-      # Disable automatic line ending conversion (again). On Windows, when we're
-      # installing dependencies, something switches the git configuration directory or
-      # re-enables autocrlf. We've not tracked down the exact cause -- and there may
-      # be multiple -- but this should ensure submodules are checked out with the
-      # appropriate line endings.
-      - name: disable git crlf conversion
-        run: src/ci/scripts/disable-git-crlf-conversion.sh
-
-      - name: ensure line endings are correct
-        run: src/ci/scripts/verify-line-endings.sh
-
-      - name: ensure backported commits are in upstream branches
-        run: src/ci/scripts/verify-backported-commits.sh
-
-      - name: ensure the stable version number is correct
-        run: src/ci/scripts/verify-stable-version-number.sh
-
-      - name: run the build
-        # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
-        run: src/ci/scripts/run-build-from-ci.sh 2>&1
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
-
-      - name: create github artifacts
-        run: src/ci/scripts/create-doc-artifacts.sh
-
-      - name: print disk usage
-        run: |
-          echo "disk usage:"
-          df -h
-
-      - name: upload artifacts to github
+      - name: Archive build output
         uses: actions/upload-artifact@v4
         with:
-          # name is set in previous step
-          name: ${{ env.DOC_ARTIFACT_NAME }}
-          path: obj/artifacts/doc
-          if-no-files-found: ignore
-          retention-days: 5
-
-      - name: upload artifacts to S3
-        run: src/ci/scripts/upload-artifacts.sh
-        env:
-          AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
-        # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
-        # builders *should* have the AWS credentials available. Still, explicitly
-        # adding the condition is helpful as this way CI will not silently skip
-        # deploying artifacts from a dist builder if the variables are misconfigured,
-        # erroring about invalid credentials instead.
-        if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
-
-      - name: upload job metrics to DataDog
-        if: needs.calculate_matrix.outputs.run_type != 'pr'
-        env:
-          DATADOG_SITE: datadoghq.com
-          DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
-          DD_GITHUB_JOB_NAME: ${{ matrix.name }}
-        run: |
-          cd src/ci
-          npm ci
-          python3 scripts/upload-build-metrics.py ../../build/cpu-usage.csv
-
-  # This job isused to tell bors the final status of the build, as there is no practical way to detect
-  # when a workflow is successful listening to webhooks only in our current bors implementation (homu).
-  outcome:
-    name: bors build finished
-    runs-on: ubuntu-24.04
-    needs: [ calculate_matrix, job ]
-    # !cancelled() executes the job regardless of whether the previous jobs passed or failed
-    if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }}
-    steps:
-      - name: checkout the source code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 2
-      # Calculate the exit status of the whole CI workflow.
-      # If all dependent jobs were successful, this exits with 0 (and the outcome job continues successfully).
-      # If a some dependent job has failed, this exits with 1.
-      - name: calculate the correct exit status
-        run: jq --exit-status 'all(.result == "success" or .result == "skipped")' <<< '${{ toJson(needs) }}'
-      # Publish the toolstate if an auto build succeeds (just before push to master)
-      - name: publish toolstate
-        run: src/ci/publish_toolstate.sh
-        shell: bash
-        if: needs.calculate_matrix.outputs.run_type == 'auto'
-        env:
-          TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
-          TOOLSTATE_PUBLISH: 1
+          name: rust-toolchain-${{ matrix.triple }}
+          path: |
+            rust/build/${{ matrix.triple }}/stage2
+            !rust/build/${{ matrix.triple }}/stage2/lib/rustlib/src
+            !rust/build/${{ matrix.triple }}/stage2/lib/rustlib/rustc-src

.github/workflows/dependencies.yml

@@ -27,7 +27,7 @@ jobs:
   not-waiting-on-bors:
     if: github.repository_owner == 'rust-lang'
     name: skip if S-waiting-on-bors
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     steps:
       - env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -47,7 +47,7 @@ jobs:
     if: github.repository_owner == 'rust-lang'
     name: update dependencies
     needs: not-waiting-on-bors
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     steps:
       - name: checkout the source code
         uses: actions/checkout@v4
@@ -56,32 +56,19 @@ jobs:
       - name: install the bootstrap toolchain
         run: |
           # Extract the stage0 version
-          TOOLCHAIN=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0)
+          TOOLCHAIN=$(jq -r '.compiler | {version,date} | join("-")' -- src/stage0.json)
           # Install and set as default
           rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
           rustup default $TOOLCHAIN
 
-      - name: cargo update compiler & tools
+      - name: cargo update
         # Remove first line that always just says "Updating crates.io index"
-        run: |
-          echo -e "\ncompiler & tools dependencies:" >> cargo_update.log
-          cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update library
-        run: |
-          echo -e "\nlibrary dependencies:" >> cargo_update.log
-          cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update rustbook
-        run: |
-          echo -e "\nrustbook dependencies:" >> cargo_update.log
-          cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
+        run: cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
       - name: upload Cargo.lock artifact for use in PR
         uses: actions/upload-artifact@v4
         with:
           name: Cargo-lock
-          path: |
-            Cargo.lock
-            library/Cargo.lock
-            src/tools/rustbook/Cargo.lock
+          path: Cargo.lock
           retention-days: 1
       - name: upload cargo-update log artifact for use in PR
         uses: actions/upload-artifact@v4
@@ -94,7 +81,7 @@ jobs:
     if: github.repository_owner == 'rust-lang'
     name: amend PR
     needs: update
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: write
       pull-requests: write
@@ -126,7 +113,7 @@ jobs:
           git config user.name github-actions
           git config user.email [email protected]
           git switch --force-create cargo_update
-          git add ./Cargo.lock ./library/Cargo.lock ./src/tools/rustbook/Cargo.lock
+          git add ./Cargo.lock
           git commit --no-verify --file=commit.txt
 
       - name: push