refactor(internal): move the os/arch detection to repo_utils (#2075)

This also changes the local_runtime_repo to explicitly check for
supported platforms instead of relying on a `None` value returned by the
helper method. This makes the behaviour exactly the same to the
behaviour before this PR and we can potentially drop the need for the
validation in the future if our local_runtime detection is more robust.

This also makes the platform detectino in `pypi_repo_utils` not depend
on `uname` and only use the `repository_ctx`. Apparently the
`module_ctx.watch` throws an error if one attempts to watch files on the
system (this is left for `repository_rule` it seems and one can only do
`module_ctx.watch` on files within the current workspace. This was
surprising, but could have been worked around by just unifying code.

This splits out things from #2059 and makes the code more succinct.

Work towards #260, #1105, #1868.

Author: aignas

python/private/local_runtime_repo.bzl

@@ -45,22 +45,6 @@ def _local_runtime_repo_impl(rctx):
     logger = repo_utils.logger(rctx)
     on_failure = rctx.attr.on_failure
 
-    platforms_os_name = repo_utils.get_platforms_os_name(rctx)
-    if not platforms_os_name:
-        if on_failure == "fail":
-            fail("Unrecognized host platform '{}': cannot determine OS constraint".format(
-                rctx.os.name,
-            ))
-
-        if on_failure == "warn":
-            logger.warn(lambda: "Unrecognized host platform '{}': cannot determine OS constraint".format(
-                rctx.os.name,
-            ))
-
-        # else, on_failure must be skip
-        rctx.file("BUILD.bazel", _expand_incompatible_template())
-        return
-
     result = _resolve_interpreter_path(rctx)
     if not result.resolved_path:
         if on_failure == "fail":

python/private/pypi/BUILD.bazel

@@ -165,6 +165,7 @@ bzl_library(
         ":requirements_files_by_platform_bzl",
         ":whl_target_platforms_bzl",
         "//python/private:normalize_name_bzl",
+        "//python/private:repo_utils_bzl",
     ],
 )
 
@@ -233,8 +234,7 @@ bzl_library(
     name = "pypi_repo_utils_bzl",
     srcs = ["pypi_repo_utils.bzl"],
     deps = [
-        "//python:versions_bzl",
-        "//python/private:toolchains_repo_bzl",
+        "//python/private:repo_utils_bzl",
     ],
 )
 

python/private/pypi/extension.bzl

@@ -199,7 +199,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
         logger = logger,
     )
 
-    repository_platform = host_platform(module_ctx.os)
+    repository_platform = host_platform(module_ctx)
     for whl_name, requirements in requirements_by_platform.items():
         # We are not using the "sanitized name" because the user
         # would need to guess what name we modified the whl name

python/private/pypi/parse_requirements.bzl

@@ -27,59 +27,11 @@ behavior.
 """
 
 load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:repo_utils.bzl", "repo_utils")
 load(":index_sources.bzl", "index_sources")
 load(":parse_requirements_txt.bzl", "parse_requirements_txt")
 load(":whl_target_platforms.bzl", "select_whls")
 
-# This includes the vendored _translate_cpu and _translate_os from
-# @platforms//host:extension.bzl at version 0.0.9 so that we don't
-# force the users to depend on it.
-
-def _translate_cpu(arch):
-    if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]:
-        return "x86_32"
-    if arch in ["amd64", "x86_64", "x64"]:
-        return "x86_64"
-    if arch in ["ppc", "ppc64", "ppc64le"]:
-        return "ppc"
-    if arch in ["arm", "armv7l"]:
-        return "arm"
-    if arch in ["aarch64"]:
-        return "aarch64"
-    if arch in ["s390x", "s390"]:
-        return "s390x"
-    if arch in ["mips64el", "mips64"]:
-        return "mips64"
-    if arch in ["riscv64"]:
-        return "riscv64"
-    return arch
-
-def _translate_os(os):
-    if os.startswith("mac os"):
-        return "osx"
-    if os.startswith("freebsd"):
-        return "freebsd"
-    if os.startswith("openbsd"):
-        return "openbsd"
-    if os.startswith("linux"):
-        return "linux"
-    if os.startswith("windows"):
-        return "windows"
-    return os
-
-# TODO @aignas 2024-05-13: consider using the same platform tags as are used in
-# the //python:versions.bzl
-DEFAULT_PLATFORMS = [
-    "linux_aarch64",
-    "linux_arm",
-    "linux_ppc",
-    "linux_s390x",
-    "linux_x86_64",
-    "osx_aarch64",
-    "osx_x86_64",
-    "windows_x86_64",
-]
-
 def parse_requirements(
         ctx,
         *,
@@ -271,20 +223,19 @@ def select_requirement(requirements, *, platform):
 
     return maybe_requirement[0]
 
-def host_platform(repository_os):
+def host_platform(ctx):
     """Return a string representation of the repository OS.
 
     Args:
-        repository_os (struct): The `module_ctx.os` or `repository_ctx.os` attribute.
-            See https://bazel.build/rules/lib/builtins/repository_os.html
+        ctx (struct): The `module_ctx` or `repository_ctx` attribute.
 
     Returns:
         The string representation of the platform that we can later used in the `pip`
         machinery.
     """
     return "{}_{}".format(
-        _translate_os(repository_os.name.lower()),
-        _translate_cpu(repository_os.arch.lower()),
+        repo_utils.get_platforms_os_name(ctx),
+        repo_utils.get_platforms_cpu_name(ctx),
     )
 
 def _add_dists(requirement, index_urls, python_version, logger = None):

python/private/pypi/pip_repository.bzl

@@ -84,7 +84,7 @@ def _pip_repository_impl(rctx):
     )
     selected_requirements = {}
     options = None
-    repository_platform = host_platform(rctx.os)
+    repository_platform = host_platform(rctx)
     for name, requirements in requirements_by_platform.items():
         r = select_requirement(
             requirements,

python/private/pypi/pypi_repo_utils.bzl

@@ -14,8 +14,7 @@
 
 ""
 
-load("//python:versions.bzl", "WINDOWS_NAME")
-load("//python/private:toolchains_repo.bzl", "get_host_os_arch")
+load("//python/private:repo_utils.bzl", "repo_utils")
 
 def _get_python_interpreter_attr(ctx, *, python_interpreter = None):
     """A helper function for getting the `python_interpreter` attribute or it's default
@@ -30,7 +29,8 @@ def _get_python_interpreter_attr(ctx, *, python_interpreter = None):
     if python_interpreter:
         return python_interpreter
 
-    if "win" in ctx.os.name:
+    os = repo_utils.get_platforms_os_name(ctx)
+    if "windows" in os:
         return "python.exe"
     else:
         return "python3"
@@ -39,7 +39,7 @@ def _resolve_python_interpreter(ctx, *, python_interpreter = None, python_interp
     """Helper function to find the python interpreter from the common attributes
 
     Args:
-        ctx: Handle to the rule repository context.
+        ctx: Handle to the rule module_ctx or repository_ctx.
         python_interpreter: The python interpreter to use.
         python_interpreter_target: The python interpreter to use after downloading the label.
 
@@ -51,11 +51,11 @@ def _resolve_python_interpreter(ctx, *, python_interpreter = None, python_interp
     if python_interpreter_target != None:
         python_interpreter = ctx.path(python_interpreter_target)
 
-        (os, _) = get_host_os_arch(ctx)
+        os = repo_utils.get_platforms_os_name(ctx)
 
         # On Windows, the symlink doesn't work because Windows attempts to find
         # Python DLLs where the symlink is, not where the symlink points.
-        if os == WINDOWS_NAME:
+        if "windows" in os:
             python_interpreter = python_interpreter.realpath
     elif "/" not in python_interpreter:
         # It's a plain command, e.g. "python3", to look up in the environment.
@@ -67,22 +67,23 @@ def _resolve_python_interpreter(ctx, *, python_interpreter = None, python_interp
         python_interpreter = ctx.path(python_interpreter)
     return python_interpreter
 
-def _construct_pypath(rctx, *, entries):
+def _construct_pypath(ctx, *, entries):
     """Helper function to construct a PYTHONPATH.
 
     Contains entries for code in this repo as well as packages downloaded from //python/pip_install:repositories.bzl.
     This allows us to run python code inside repository rule implementations.
 
     Args:
-        rctx: Handle to the repository_context.
+        ctx: Handle to the module_ctx or repository_ctx.
         entries: The list of entries to add to PYTHONPATH.
 
     Returns: String of the PYTHONPATH.
     """
 
-    separator = ":" if not "windows" in rctx.os.name.lower() else ";"
+    os = repo_utils.get_platforms_os_name(ctx)
+    separator = ";" if "windows" in os else ":"
     pypath = separator.join([
-        str(rctx.path(entry).dirname)
+        str(ctx.path(entry).dirname)
         # Use a dict as a way to remove duplicates and then sort it.
         for entry in sorted({x: None for x in entries})
     ])

python/private/repo_utils.bzl

@@ -341,25 +341,60 @@ def _outputs_to_str(result):
             lines.append("<{} empty>".format(name))
     return "\n".join(lines)
 
+# This includes the vendored _translate_cpu and _translate_os from
+# @platforms//host:extension.bzl at version 0.0.9 so that we don't
+# force the users to depend on it.
+
 def _get_platforms_os_name(rctx):
     """Return the name in @platforms//os for the host os.
 
     Args:
         rctx: repository_ctx
 
     Returns:
-        `str | None`. The target name if it maps to known platforms
-        value, otherwise None.
+        `str`. The target name.
     """
     os = rctx.os.name.lower()
-    if "linux" in os:
-        return os
-    if "windows" in os:
-        return "windows"
-    if "mac" in os:
+
+    if os.startswith("mac os"):
         return "osx"
+    if os.startswith("freebsd"):
+        return "freebsd"
+    if os.startswith("openbsd"):
+        return "openbsd"
+    if os.startswith("linux"):
+        return "linux"
+    if os.startswith("windows"):
+        return "windows"
+    return os
+
+def _get_platforms_cpu_name(rctx):
+    """Return the name in @platforms//cpu for the host arch.
 
-    return None
+    Args:
+        rctx: repository_ctx
+
+    Returns:
+        `str`. The target name.
+    """
+    arch = rctx.os.arch.lower()
+    if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]:
+        return "x86_32"
+    if arch in ["amd64", "x86_64", "x64"]:
+        return "x86_64"
+    if arch in ["ppc", "ppc64", "ppc64le"]:
+        return "ppc"
+    if arch in ["arm", "armv7l"]:
+        return "arm"
+    if arch in ["aarch64"]:
+        return "aarch64"
+    if arch in ["s390x", "s390"]:
+        return "s390x"
+    if arch in ["mips64el", "mips64"]:
+        return "mips64"
+    if arch in ["riscv64"]:
+        return "riscv64"
+    return arch
 
 # TODO: Remove after Bazel 6 support dropped
 def _watch(rctx, *args, **kwargs):
@@ -379,6 +414,7 @@ repo_utils = struct(
     execute_checked = _execute_checked,
     execute_checked_stdout = _execute_checked_stdout,
     execute_unchecked = _execute_unchecked,
+    get_platforms_cpu_name = _get_platforms_cpu_name,
     get_platforms_os_name = _get_platforms_os_name,
     getenv = _getenv,
     is_repo_debug_enabled = _is_repo_debug_enabled,