refactor: make env marker config available through target and flag (#2853)

This factors creation of (most of) the env marker dict into a separate
target and
provides a label flag to allow customizing the target that provides it.

This makes it easier for users to override how env marker values are
computed. The
`env_marker_setting` rule will still, if necessary, compute values from
the toolchain,
but existing keys (computed from the env marker config target) have
precedence.

The `EnvMarkerInfo` provider is the interface for implementing a custom
env marker
config target; it will be publically exposed in a subsequent PR.

Along the way, unify how the env dict and defaults are set.

Work towards #2826

Author: rickeylev

docs/api/rules_python/python/config_settings/index.md

@@ -159,6 +159,18 @@ Values:
 :::
 ::::
 
+::::{bzl:flag} pip_env_marker_config
+The target that provides the values for pip env marker evaluation.
+
+Default: `//python/config_settings:_pip_env_marker_default_config`
+
+This flag points to a target providing {obj}`EnvMarkerInfo`, which determines
+the values used when environment markers are resolved at build time.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
+::::
+
 ::::{bzl:flag} pip_whl
 Set what distributions are used in the `pip` integration.
 

docs/pypi-dependencies.md

@@ -338,7 +338,6 @@ leg of the dependency manually. For instance by making
 perhaps `apache-airflow-providers-common-sql`.
 
 
-(bazel-downloader)=
 ### Multi-platform support
 
 Multi-platform support of cross-building the wheels can be done in two ways - either
@@ -391,6 +390,31 @@ compatible indexes.
 This is only supported on `bzlmd`.
 ```
 
+<!--
+
+TODO: uncomment this when analysis-phase dependency selection is available
+
+#### Customizing requirements resolution
+
+In Python packaging, packages can express dependencies with conditions
+using "environment markers", which represent the Python version, OS, etc.
+
+While the PyPI integration provides reasonable defaults to support most
+platforms and environment markers, the values it uses can be customized in case
+more esoteric configurations are needed.
+
+To customize the values used, you need to do two things:
+1. Define a target that returns {obj}`EnvMarkerInfo`
+2. Set the {obj}`//python/config_settings:pip_env_marker_config` flag to
+   the target defined in (1).
+
+The keys and values should be compatible with the [PyPA dependency specifiers
+specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/).
+This is not strictly enforced, however, so you can return a subset of keys or
+additional keys, which become available during dependency evalution.
+
+-->
+
 (bazel-downloader)=
 ### Bazel downloader and multi-platform wheel hub repository.
 
@@ -487,3 +511,9 @@ Bazel will call this file like `cred_helper.sh get` and use the returned JSON to
 into whatever HTTP(S) request it performs against `example.com`.
 
 [rfc7617]: https://datatracker.ietf.org/doc/html/rfc7617
+
+<!--
+
+
+
+-->

python/config_settings/BUILD.bazel

@@ -220,3 +220,10 @@ string_flag(
 define_pypi_internal_flags(
     name = "define_pypi_internal_flags",
 )
+
+label_flag(
+    name = "pip_env_marker_config",
+    build_setting_default = ":_pip_env_marker_default_config",
+    # NOTE: Only public because it is used in pip hub repos.
+    visibility = ["//visibility:public"],
+)

python/private/pypi/BUILD.bazel

@@ -71,6 +71,23 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "env_marker_info_bzl",
+    srcs = ["env_marker_info.bzl"],
+)
+
+bzl_library(
+    name = "env_marker_setting_bzl",
+    srcs = ["env_marker_setting.bzl"],
+    deps = [
+        ":env_marker_info_bzl",
+        ":pep508_env_bzl",
+        ":pep508_evaluate_bzl",
+        "//python/private:toolchain_types_bzl",
+        "@bazel_skylib//rules:common_settings",
+    ],
+)
+
 bzl_library(
     name = "evaluate_markers_bzl",
     srcs = ["evaluate_markers.bzl"],
@@ -111,6 +128,8 @@ bzl_library(
     name = "flags_bzl",
     srcs = ["flags.bzl"],
     deps = [
+        ":env_marker_info.bzl",
+        ":pep508_env_bzl",
         "//python/private:enum_bzl",
         "@bazel_skylib//rules:common_settings",
     ],

python/private/pypi/env_marker_info.bzl

@@ -0,0 +1,26 @@
+"""Provider for implementing environment marker values."""
+
+EnvMarkerInfo = provider(
+    doc = """
+The values to use during environment marker evaluation.
+
+:::{seealso}
+The {obj}`--//python/config_settings:pip_env_marker_config` flag.
+:::
+
+:::{versionadded} VERSION_NEXT_FEATURE
+""",
+    fields = {
+        "env": """
+:type: dict[str, str]
+
+The values to use for environment markers when evaluating an expression.
+
+The keys and values should be compatible with the [PyPA dependency specifiers
+specification](https://packaging.python.org/en/latest/specifications/dependency-specifiers/)
+
+Missing values will be set to the specification's defaults or computed using
+available toolchain information.
+""",
+    },
+)

python/private/pypi/env_marker_setting.bzl

@@ -2,14 +2,8 @@
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE")
-load(
-    ":pep508_env.bzl",
-    "env_aliases",
-    "os_name_select_map",
-    "platform_machine_select_map",
-    "platform_system_select_map",
-    "sys_platform_select_map",
-)
+load(":env_marker_info.bzl", "EnvMarkerInfo")
+load(":pep508_env.bzl", "create_env", "set_missing_env_defaults")
 load(":pep508_evaluate.bzl", "evaluate")
 
 # Use capitals to hint its not an actual boolean type.
@@ -39,72 +33,37 @@ def env_marker_setting(*, name, expression, **kwargs):
     _env_marker_setting(
         name = name,
         expression = expression,
-        os_name = select(os_name_select_map),
-        sys_platform = select(sys_platform_select_map),
-        platform_machine = select(platform_machine_select_map),
-        platform_system = select(platform_system_select_map),
-        platform_release = select({
-            "@platforms//os:osx": "USE_OSX_VERSION_FLAG",
-            "//conditions:default": "",
-        }),
         **kwargs
     )
 
 def _env_marker_setting_impl(ctx):
-    env = {}
+    env = create_env()
+    env.update(
+        ctx.attr._env_marker_config_flag[EnvMarkerInfo].env,
+    )
 
     runtime = ctx.toolchains[TARGET_TOOLCHAIN_TYPE].py3_runtime
-    if runtime.interpreter_version_info:
-        version_info = runtime.interpreter_version_info
-        env["python_version"] = "{major}.{minor}".format(
-            major = version_info.major,
-            minor = version_info.minor,
-        )
-        full_version = _format_full_version(version_info)
-        env["python_full_version"] = full_version
-        env["implementation_version"] = full_version
-    else:
-        env["python_version"] = _get_flag(ctx.attr._python_version_major_minor_flag)
-        full_version = _get_flag(ctx.attr._python_full_version_flag)
-        env["python_full_version"] = full_version
-        env["implementation_version"] = full_version
-
-    # We assume cpython if the toolchain doesn't specify because it's most
-    # likely to be true.
-    env["implementation_name"] = runtime.implementation_name or "cpython"
-    env["os_name"] = ctx.attr.os_name
-    env["sys_platform"] = ctx.attr.sys_platform
-    env["platform_machine"] = ctx.attr.platform_machine
-
-    # The `platform_python_implementation` marker value is supposed to come
-    # from `platform.python_implementation()`, however, PEP 421 introduced
-    # `sys.implementation.name` and the `implementation_name` env marker to
-    # replace it. Per the platform.python_implementation docs, there's now
-    # essentially just two possible "registered" values: CPython or PyPy.
-    # Rather than add a field to the toolchain, we just special case the value
-    # from `sys.implementation.name` to handle the two documented values.
-    platform_python_impl = runtime.implementation_name
-    if platform_python_impl == "cpython":
-        platform_python_impl = "CPython"
-    elif platform_python_impl == "pypy":
-        platform_python_impl = "PyPy"
-    env["platform_python_implementation"] = platform_python_impl
-
-    # NOTE: Platform release for Android will be Android version:
-    # https://peps.python.org/pep-0738/#platform
-    # Similar for iOS:
-    # https://peps.python.org/pep-0730/#platform
-    platform_release = ctx.attr.platform_release
-    if platform_release == "USE_OSX_VERSION_FLAG":
-        platform_release = _get_flag(ctx.attr._pip_whl_osx_version_flag)
-    env["platform_release"] = platform_release
-    env["platform_system"] = ctx.attr.platform_system
-
-    # For lack of a better option, just use an empty string for now.
-    env["platform_version"] = ""
-
-    env.update(env_aliases())
 
+    if "python_version" not in env:
+        if runtime.interpreter_version_info:
+            version_info = runtime.interpreter_version_info
+            env["python_version"] = "{major}.{minor}".format(
+                major = version_info.major,
+                minor = version_info.minor,
+            )
+            full_version = _format_full_version(version_info)
+            env["python_full_version"] = full_version
+            env["implementation_version"] = full_version
+        else:
+            env["python_version"] = _get_flag(ctx.attr._python_version_major_minor_flag)
+            full_version = _get_flag(ctx.attr._python_full_version_flag)
+            env["python_full_version"] = full_version
+            env["implementation_version"] = full_version
+
+    if "implementation_name" not in env and runtime.implementation_name:
+        env["implementation_name"] = runtime.implementation_name
+
+    set_missing_env_defaults(env)
     if evaluate(ctx.attr.expression, env = env):
         value = _ENV_MARKER_TRUE
     else:
@@ -125,14 +84,9 @@ for the specification of behavior.
             mandatory = True,
             doc = "Environment marker expression to evaluate.",
         ),
-        "os_name": attr.string(),
-        "platform_machine": attr.string(),
-        "platform_release": attr.string(),
-        "platform_system": attr.string(),
-        "sys_platform": attr.string(),
-        "_pip_whl_osx_version_flag": attr.label(
-            default = "//python/config_settings:pip_whl_osx_version",
-            providers = [[BuildSettingInfo], [config_common.FeatureFlagInfo]],
+        "_env_marker_config_flag": attr.label(
+            default = "//python/config_settings:pip_env_marker_config",
+            providers = [EnvMarkerInfo],
         ),
         "_python_full_version_flag": attr.label(
             default = "//python/config_settings:python_version",

python/private/pypi/flags.bzl

@@ -20,6 +20,15 @@ unnecessary files when all that are needed are flag definitions.
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo", "string_flag")
 load("//python/private:enum.bzl", "enum")
+load(":env_marker_info.bzl", "EnvMarkerInfo")
+load(
+    ":pep508_env.bzl",
+    "create_env",
+    "os_name_select_map",
+    "platform_machine_select_map",
+    "platform_system_select_map",
+    "sys_platform_select_map",
+)
 
 # Determines if we should use whls for third party
 #
@@ -82,6 +91,10 @@ def define_pypi_internal_flags(name):
         visibility = ["//visibility:public"],
     )
 
+    _default_env_marker_config(
+        name = "_pip_env_marker_default_config",
+    )
+
 def _allow_wheels_flag_impl(ctx):
     input = ctx.attr._setting[BuildSettingInfo].value
     value = "yes" if input in ["auto", "only"] else "no"
@@ -97,3 +110,58 @@ This rule allows us to greatly reduce the number of config setting targets at no
 if we are duplicating some of the functionality of the `native.config_setting`.
 """,
 )
+
+def _default_env_marker_config(**kwargs):
+    _env_marker_config(
+        os_name = select(os_name_select_map),
+        sys_platform = select(sys_platform_select_map),
+        platform_machine = select(platform_machine_select_map),
+        platform_system = select(platform_system_select_map),
+        platform_release = select({
+            "@platforms//os:osx": "USE_OSX_VERSION_FLAG",
+            "//conditions:default": "",
+        }),
+        **kwargs
+    )
+
+def _env_marker_config_impl(ctx):
+    env = create_env()
+    env["os_name"] = ctx.attr.os_name
+    env["sys_platform"] = ctx.attr.sys_platform
+    env["platform_machine"] = ctx.attr.platform_machine
+
+    # NOTE: Platform release for Android will be Android version:
+    # https://peps.python.org/pep-0738/#platform
+    # Similar for iOS:
+    # https://peps.python.org/pep-0730/#platform
+    platform_release = ctx.attr.platform_release
+    if platform_release == "USE_OSX_VERSION_FLAG":
+        platform_release = _get_flag(ctx.attr._pip_whl_osx_version_flag)
+    env["platform_release"] = platform_release
+    env["platform_system"] = ctx.attr.platform_system
+
+    # NOTE: We intentionally do not call set_missing_env_defaults() here because
+    # `env_marker_setting()` computes missing values using the toolchain.
+    return [EnvMarkerInfo(env = env)]
+
+_env_marker_config = rule(
+    implementation = _env_marker_config_impl,
+    attrs = {
+        "os_name": attr.string(),
+        "platform_machine": attr.string(),
+        "platform_release": attr.string(),
+        "platform_system": attr.string(),
+        "sys_platform": attr.string(),
+        "_pip_whl_osx_version_flag": attr.label(
+            default = "//python/config_settings:pip_whl_osx_version",
+            providers = [[BuildSettingInfo], [config_common.FeatureFlagInfo]],
+        ),
+    },
+)
+
+def _get_flag(t):
+    if config_common.FeatureFlagInfo in t:
+        return t[config_common.FeatureFlagInfo].value
+    if BuildSettingInfo in t:
+        return t[BuildSettingInfo].value
+    fail("Should not occur: {} does not have necessary providers")