Generate the Docker CI matrix

.github/workflows/linux.yml

@@ -54,14 +54,9 @@ jobs:
       - generate-matrix
     strategy:
       fail-fast: false
-      matrix:
-        image:
-          - build
-          - build.cross
-          - build.cross-riscv64
-          - gcc
-    name: ${{ matrix.image }}
-    runs-on: depot-ubuntu-22.04
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.docker-build-matrix) }}
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.runner }}
     permissions:
       packages: write
     steps:
@@ -95,37 +90,38 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: .
-          file: build/${{ matrix.image }}.Dockerfile
+          file: build/${{ matrix.name }}.Dockerfile
           labels: org.opencontainers.image.source=https://github.com/${{ env.REPO_NAME }}
           # Cache from/to the current branch of the current repo as the primary cache key.
           # Cache from the default branch of the current repo so branches can have cache hits.
           # Cache from the default branch of the canonical repo so forks can have cache hits.
           # Ignore errors on cache writes so CI of forks works without a valid GHCR config.
           cache-from: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }}
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-main
-            type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.image }}-main
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-${{ env.GIT_REF_NAME }}
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-main
+            type=registry,ref=ghcr.io/astral-sh/python-build-standalone:${{ matrix.name }}-main
           cache-to: |
-            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.image }}-${{ env.GIT_REF_NAME }},ignore-error=true
+            type=registry,ref=ghcr.io/${{ env.REPO_NAME }}:${{ matrix.name }}-${{ env.GIT_REF_NAME }},ignore-error=true
           outputs: |
-            type=docker,dest=build/image-${{ matrix.image }}.tar
+            type=docker,dest=build/image-${{ matrix.name }}.tar
 
       - name: Compress Image
         run: |
-          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.image }}
+          echo ${{ steps.build-image.outputs.imageid }} > build/image-${{ matrix.name }}
           zstd -v -T0 -6 --rm build/image-*.tar
 
       - name: Upload Docker Image
         uses: actions/upload-artifact@v4
         with:
-          name: image-${{ matrix.image }}
+          name: image-${{ matrix.name }}
           path: build/image-*
 
   generate-matrix:
     runs-on: ubuntu-latest
     outputs:
-      matrix-0: ${{ steps.set-matrix.outputs.matrix-0 }}
-      matrix-1: ${{ steps.set-matrix.outputs.matrix-1 }}
+      python-build-matrix-0: ${{ steps.set-matrix.outputs.python-build-matrix-0 }}
+      python-build-matrix-1: ${{ steps.set-matrix.outputs.python-build-matrix-1 }}
+      docker-build-matrix: ${{ steps.set-matrix.outputs.docker-build-matrix }}
       any_builds: ${{ steps.set-matrix.outputs.any_builds }}
       pythonbuild_changed: ${{ steps.check-pythonbuild.outputs.changed }}
     steps:
@@ -152,13 +148,14 @@ jobs:
             --max-shards 2 \
             > matrix.json
 
-          echo "matrix-0=$(jq -c '.["0"]' matrix.json)" >> $GITHUB_OUTPUT
-          echo "matrix-1=$(jq -c '.["1"]' matrix.json)" >> $GITHUB_OUTPUT
+          echo "python-build-matrix-0=$(jq -c '."python-build"["0"]' matrix.json)" >> $GITHUB_OUTPUT
+          echo "python-build-matrix-1=$(jq -c '."python-build"["1"]' matrix.json)" >> $GITHUB_OUTPUT
+          echo "docker-build-matrix=$(jq -c '."docker-build"' matrix.json)" >> $GITHUB_OUTPUT
 
           # Display the matrix for debugging too
           cat matrix.json | jq
 
-          if jq -e '.["0"].include | length > 0' matrix.json > /dev/null; then
+          if jq -e '."python-build"["0"].include | length > 0' matrix.json > /dev/null; then
             # Build matrix has entries
             echo "any_builds=true" >> $GITHUB_OUTPUT
           else
@@ -189,7 +186,7 @@ jobs:
       attestations: write
     runs-on: ${{ matrix.runner }}
     strategy:
-      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-0) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.python-build-matrix-0) }}
       fail-fast: false
     name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
     steps:
@@ -289,7 +286,7 @@ jobs:
       attestations: write
     runs-on: ${{ matrix.runner }}
     strategy:
-      matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix-1) }}
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.python-build-matrix-1) }}
       fail-fast: false
     name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
     steps:

.github/workflows/macos.yml

@@ -67,11 +67,15 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json && echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+          uv run ci-matrix.py --platform darwin --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
+
+          # Extract python-build matrix
+          echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
+
           # Display the matrix for debugging too
           cat matrix.json | jq
 
-          if jq -e '.include | length > 0' matrix.json > /dev/null; then
+          if jq -e '."python-build".include | length > 0' matrix.json > /dev/null; then
             # Build matrix has entries
             echo "any_builds=true" >> $GITHUB_OUTPUT
           else

.github/workflows/windows.yml

@@ -67,11 +67,15 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json && echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+          uv run ci-matrix.py --platform windows --labels '${{ steps.get-labels.outputs.labels }}' > matrix.json
+
+          # Extract python-build matrix
+          echo "matrix=$(jq -c '."python-build"' matrix.json)" >> $GITHUB_OUTPUT
+
           # Display the matrix for debugging too
           cat matrix.json | jq
 
-          if jq -e '.include | length > 0' matrix.json > /dev/null; then
+          if jq -e '."python-build".include | length > 0' matrix.json > /dev/null; then
             # Build matrix has entries
             echo "any_builds=true" >> $GITHUB_OUTPUT
           else

ci-matrix.py

@@ -19,6 +19,14 @@
 CI_EXTRA_SKIP_LABELS = ["documentation"]
 CI_MATRIX_SIZE_LIMIT = 256  # The maximum size of a matrix in GitHub Actions
 
+# Docker images for building toolchains and dependencies
+DOCKER_BUILD_IMAGES = [
+    {"name": "build", "arch": "x86_64"},
+    {"name": "build.cross", "arch": "x86_64"},
+    {"name": "build.cross-riscv64", "arch": "x86_64"},
+    {"name": "gcc", "arch": "x86_64"},
+]
+
 
 def meets_conditional_version(version: str, min_version: str) -> bool:
     return Version(version) >= Version(min_version)
@@ -89,26 +97,50 @@ def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) ->
     return True
 
 
-def generate_matrix_entries(
+def generate_docker_matrix_entries(
+    runners: dict[str, Any],
+    platform_filter: Optional[str] = None,
+) -> list[dict[str, str]]:
+    """Generate matrix entries for docker image builds."""
+    if platform_filter and platform_filter != "linux":
+        return []
+
+    matrix_entries = []
+    for image in DOCKER_BUILD_IMAGES:
+        # Find appropriate runner for Linux platform with the specified architecture
+        runner = find_runner(runners, "linux", image["arch"])
+
+        entry = {
+            "name": image["name"],
+            "arch": image["arch"],
+            "runner": runner,
+        }
+        matrix_entries.append(entry)
+
+    return matrix_entries
+
+
+def generate_python_build_matrix_entries(
     config: dict[str, Any],
     runners: dict[str, Any],
     platform_filter: Optional[str] = None,
     label_filters: Optional[dict[str, set[str]]] = None,
 ) -> list[dict[str, str]]:
+    """Generate matrix entries for python builds."""
     matrix_entries = []
 
     for platform, platform_config in config.items():
         if platform_filter and platform != platform_filter:
             continue
 
         for target_triple, target_config in platform_config.items():
-            add_matrix_entries_for_config(
+            add_python_build_entries_for_config(
                 matrix_entries,
                 target_triple,
                 target_config,
                 platform,
                 runners,
-                label_filters.get("directives", set()),
+                label_filters.get("directives", set()) if label_filters else set(),
             )
 
     # Apply label filters if present
@@ -144,14 +176,15 @@ def find_runner(runners: dict[str, Any], platform: str, arch: str) -> str:
     raise RuntimeError(f"No runner found for platform {platform!r} and arch {arch!r}")
 
 
-def add_matrix_entries_for_config(
+def add_python_build_entries_for_config(
     matrix_entries: list[dict[str, str]],
     target_triple: str,
     config: dict[str, Any],
     platform: str,
     runners: dict[str, Any],
     directives: set[str],
 ) -> None:
+    """Add python build matrix entries for a specific target configuration."""
     python_versions = config["python_versions"]
     build_options = config["build_options"]
     arch = config["arch"]
@@ -233,6 +266,12 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="If only free runners should be used.",
     )
+    parser.add_argument(
+        "--matrix-type",
+        choices=["python-build", "docker-build", "all"],
+        default="all",
+        help="Which matrix types to generate (default: all)",
+    )
     return parser.parse_args()
 
 
@@ -254,36 +293,59 @@ def main() -> None:
             if runner_config.get("free")
         }
 
-    entries = generate_matrix_entries(
-        config,
-        runners,
-        args.platform,
-        labels,
-    )
-
-    if args.max_shards:
-        matrix = {}
-        shards = (len(entries) // CI_MATRIX_SIZE_LIMIT) + 1
-        if shards > args.max_shards:
-            print(
-                f"error: matrix of size {len(entries)} requires {shards} shards, but the maximum is {args.max_shards}; consider increasing `--max-shards`",
-                file=sys.stderr,
-            )
-            sys.exit(1)
-        for shard in range(args.max_shards):
-            shard_entries = entries[
-                shard * CI_MATRIX_SIZE_LIMIT : (shard + 1) * CI_MATRIX_SIZE_LIMIT
-            ]
-            matrix[str(shard)] = {"include": shard_entries}
-    else:
-        if len(entries) > CI_MATRIX_SIZE_LIMIT:
-            print(
-                f"warning: matrix of size {len(entries)} exceeds limit of {CI_MATRIX_SIZE_LIMIT} but sharding is not enabled; consider setting `--max-shards`",
-                file=sys.stderr,
+    result = {}
+
+    # Generate python-build matrix if requested
+    python_entries = []
+    if args.matrix_type in ["python-build", "all"]:
+        python_entries = generate_python_build_matrix_entries(
+            config,
+            runners,
+            args.platform,
+            labels,
+        )
+
+        if args.max_shards:
+            python_build_matrix = {}
+            shards = (len(python_entries) // CI_MATRIX_SIZE_LIMIT) + 1
+            if shards > args.max_shards:
+                print(
+                    f"error: python-build matrix of size {len(python_entries)} requires {shards} shards, but the maximum is {args.max_shards}; consider increasing `--max-shards`",
+                    file=sys.stderr,
+                )
+                sys.exit(1)
+            for shard in range(args.max_shards):
+                shard_entries = python_entries[
+                    shard * CI_MATRIX_SIZE_LIMIT : (shard + 1) * CI_MATRIX_SIZE_LIMIT
+                ]
+                python_build_matrix[str(shard)] = {"include": shard_entries}
+            result["python-build"] = python_build_matrix
+        else:
+            if len(python_entries) > CI_MATRIX_SIZE_LIMIT:
+                print(
+                    f"warning: python-build matrix of size {len(python_entries)} exceeds limit of {CI_MATRIX_SIZE_LIMIT} but sharding is not enabled; consider setting `--max-shards`",
+                    file=sys.stderr,
+                )
+            result["python-build"] = {"include": python_entries}
+
+    # Generate docker-build matrix if requested
+    # Only include docker builds if there are Linux python builds
+    if args.matrix_type in ["docker-build", "all"]:
+        # Check if we have any Linux python builds
+        has_linux_builds = any(
+            entry.get("platform") == "linux" for entry in python_entries
+        )
+
+        # If no platform filter or explicitly requesting docker-build only, include docker builds
+        # Otherwise, only include if there are Linux python builds
+        if args.matrix_type == "docker-build" or has_linux_builds:
+            docker_entries = generate_docker_matrix_entries(
+                runners,
+                args.platform,
             )
-        matrix = {"include": entries}
+            result["docker-build"] = {"include": docker_entries}
 
-    print(json.dumps(matrix))
+    print(json.dumps(result))
 
 
 if __name__ == "__main__":