Generate CI runners for build matrix dynamically (#483)

Author: zanieb

.github/workflows/linux.yml

@@ -168,7 +168,7 @@ jobs:
       - generate-matrix
       - pythonbuild
       - image
-    runs-on: depot-ubuntu-22.04
+    runs-on: ${{ matrix.runner }}
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
       fail-fast: false

.github/workflows/macos.yml

@@ -90,7 +90,7 @@ jobs:
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
       fail-fast: false
-    runs-on: depot-macos-latest
+    runs-on: ${{ matrix.runner }}
     name: ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
     steps:
       - uses: actions/checkout@v4

.github/workflows/windows.yml

@@ -88,7 +88,7 @@ jobs:
     needs:
       - generate-matrix
       - pythonbuild
-    runs-on: windows-latest-large
+    runs-on: ${{ matrix.runner }}
     strategy:
       matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }}
       fail-fast: false

ci-matrix.py

@@ -14,6 +14,7 @@
 from packaging.version import Version
 
 CI_TARGETS_YAML = "ci-targets.yaml"
+CI_RUNNERS_YAML = "ci-runners.yaml"
 CI_EXTRA_SKIP_LABELS = ["documentation"]
 
 
@@ -88,6 +89,7 @@ def should_include_entry(entry: dict[str, str], filters: dict[str, set[str]]) ->
 
 def generate_matrix_entries(
     config: dict[str, Any],
+    runners: dict[str, Any],
     platform_filter: Optional[str] = None,
     label_filters: Optional[dict[str, set[str]]] = None,
 ) -> list[dict[str, str]]:
@@ -103,6 +105,7 @@ def generate_matrix_entries(
                 target_triple,
                 target_config,
                 platform,
+                runners,
                 label_filters.get("directives", set()),
             )
 
@@ -117,22 +120,50 @@ def generate_matrix_entries(
     return matrix_entries
 
 
+def find_runner(runners: dict[str, Any], platform: str, arch: str) -> str:
+    # Find a matching platform first
+    match_platform = [
+        runner for runner in runners if runners[runner]["platform"] == platform
+    ]
+
+    # Then, find a matching architecture
+    match_arch = [
+        runner for runner in match_platform if runners[runner]["arch"] == arch
+    ]
+
+    # If there's a matching architecture, use that
+    if match_arch:
+        return match_arch[0]
+
+    # Otherwise, use the first with a matching platform
+    if match_platform:
+        return match_platform[0]
+
+    raise RuntimeError(f"No runner found for platform {platform!r} and arch {arch!r}")
+
+
 def add_matrix_entries_for_config(
     matrix_entries: list[dict[str, str]],
     target_triple: str,
     config: dict[str, Any],
     platform: str,
+    runners: dict[str, Any],
     directives: set[str],
 ) -> None:
     python_versions = config["python_versions"]
     build_options = config["build_options"]
     arch = config["arch"]
+    runner = find_runner(runners, platform, arch)
 
     # Create base entry that will be used for all variants
     base_entry = {
         "arch": arch,
         "target_triple": target_triple,
         "platform": platform,
+        "runner": runner,
+        # If `run` is in the config, use that — otherwise, default to if the
+        # runner architecture matches the build architecture
+        "run": str(config.get("run", runners[runner]["arch"] == arch)).lower(),
     }
 
     # Add optional fields if they exist
@@ -142,8 +173,6 @@ def add_matrix_entries_for_config(
         base_entry["libc"] = config["libc"]
     if "vcvars" in config:
         base_entry["vcvars"] = config["vcvars"]
-    if "run" in config:
-        base_entry["run"] = str(config["run"]).lower()
 
     if "dry-run" in directives:
         base_entry["dry-run"] = "true"
@@ -191,6 +220,11 @@ def parse_args() -> argparse.Namespace:
         "--labels",
         help="Comma-separated list of labels to filter by (e.g., 'platform:darwin,python:3.13,build:debug'), all must match.",
     )
+    parser.add_argument(
+        "--free-runners",
+        action="store_true",
+        help="If only free runners should be used.",
+    )
     return parser.parse_args()
 
 
@@ -201,9 +235,21 @@ def main() -> None:
     with open(CI_TARGETS_YAML, "r") as f:
         config = yaml.safe_load(f)
 
+    with open(CI_RUNNERS_YAML, "r") as f:
+        runners = yaml.safe_load(f)
+
+    # If only free runners are allowed, reduce to a subset
+    if args.free_runners:
+        runners = {
+            runner: runner_config
+            for runner, runner_config in runners.items()
+            if runner_config.get("free")
+        }
+
     matrix = {
         "include": generate_matrix_entries(
             config,
+            runners,
             args.platform,
             labels,
         )

ci-runners.yaml

@@ -0,0 +1,37 @@
+# Describes the runners that the CI system can use
+
+depot-ubuntu-22.04:
+  arch: x86_64
+  platform: linux
+  free: false
+
+# TODO: Enable this runner to perform native builds for aarch64
+# depot-ubuntu-22.04-arm:
+#   arch: aarch64
+#   platform: linux
+#   free: false
+
+depot-macos-latest:
+  arch: x86_64
+  platform: darwin
+  free: false
+
+ubuntu-latest:
+  arch: x86_64
+  platform: linux
+  free: true
+
+macos-latest:
+  arch: x86_64
+  platform: darwin
+  free: true
+
+windows-latest-large:
+  arch: x86_64
+  platform: windows
+  free: false
+
+windows-latest:
+  arch: x86_64
+  platform: windows
+  free: true