fix(pypi): call python --version before marker eval (#2819)

`bzlmod` has the full python version information statically and we don't
need to call Python to get its version, but for `WORKSPACE` that is not
the case and we have to call it before evaluating the markers in
universal requirements files.

This also fixes transitions in the `compile_pip_requirements` macro
where
the `.update` target would not transition correctly based on the
`python_version` parameter.

Fixes #2818

Author: aignas

CHANGELOG.md

@@ -151,6 +151,10 @@ END_UNRELEASED_TEMPLATE
 * (pypi) Correctly handle `METADATA` entries when `python_full_version` is used in
   the environment marker.
   Fixes [#2319](https://github.com/bazel-contrib/rules_python/issues/2319).
+* (pypi) Correctly handle `python_version` parameter and transition the requirement
+  locking to the right interpreter version when using
+  {obj}`compile_pip_requirements` rule.
+  See [#2819](https://github.com/bazel-contrib/rules_python/pull/2819).
 
 {#1-4-0-added}
 ### Added

examples/multi_python_versions/requirements/requirements.in

@@ -1 +1 @@
-websockets
+websockets ; python_full_version > "3.9.1"

examples/multi_python_versions/requirements/requirements_lock_3_10.txt

@@ -4,7 +4,7 @@
 #
 #    bazel run //requirements:requirements_3_10.update
 #
-websockets==11.0.3 \
+websockets==11.0.3 ; python_full_version > "3.9.1" \
     --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \
     --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \
     --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \

examples/multi_python_versions/requirements/requirements_lock_3_11.txt

@@ -4,7 +4,7 @@
 #
 #    bazel run //requirements:requirements_3_11.update
 #
-websockets==11.0.3 \
+websockets==11.0.3 ; python_full_version > "3.9.1" \
     --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \
     --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \
     --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \

examples/multi_python_versions/requirements/requirements_lock_3_9.txt

@@ -4,7 +4,7 @@
 #
 #    bazel run //requirements:requirements_3_9.update
 #
-websockets==11.0.3 \
+websockets==11.0.3 ; python_full_version > "3.9.1" \
     --hash=sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd \
     --hash=sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f \
     --hash=sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998 \

python/private/pypi/BUILD.bazel

@@ -283,6 +283,7 @@ bzl_library(
         ":evaluate_markers_bzl",
         ":parse_requirements_bzl",
         ":pip_repository_attrs_bzl",
+        ":pypi_repo_utils_bzl",
         ":render_pkg_aliases_bzl",
         ":whl_config_setting_bzl",
         "//python/private:normalize_name_bzl",

python/private/pypi/evaluate_markers.bzl

@@ -19,11 +19,12 @@ load(":pep508_evaluate.bzl", "evaluate")
 load(":pep508_platform.bzl", "platform_from_str")
 load(":pep508_requirement.bzl", "requirement")
 
-def evaluate_markers(requirements):
+def evaluate_markers(requirements, python_version = None):
     """Return the list of supported platforms per requirements line.
 
     Args:
-        requirements: dict[str, list[str]] of the requirement file lines to evaluate.
+        requirements: {type}`dict[str, list[str]]` of the requirement file lines to evaluate.
+        python_version: {type}`str | None` the version that can be used when evaluating the markers.
 
     Returns:
         dict of string lists with target platforms
@@ -32,7 +33,7 @@ def evaluate_markers(requirements):
     for req_string, platforms in requirements.items():
         req = requirement(req_string)
         for platform in platforms:
-            if evaluate(req.marker, env = env(platform_from_str(platform, None))):
+            if evaluate(req.marker, env = env(platform_from_str(platform, python_version))):
                 ret.setdefault(req_string, []).append(platform)
 
     return ret

python/private/pypi/pip_compile.bzl

@@ -160,6 +160,7 @@ def pip_compile(
     py_binary(
         name = name + ".update",
         env = env,
+        python_version = kwargs.get("python_version", None),
         **attrs
     )
 

python/private/pypi/pip_repository.bzl

@@ -16,11 +16,12 @@
 
 load("@bazel_skylib//lib:sets.bzl", "sets")
 load("//python/private:normalize_name.bzl", "normalize_name")
-load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR")
+load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils")
 load("//python/private:text_util.bzl", "render")
 load(":evaluate_markers.bzl", "evaluate_markers")
 load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
 load(":pip_repository_attrs.bzl", "ATTRS")
+load(":pypi_repo_utils.bzl", "pypi_repo_utils")
 load(":render_pkg_aliases.bzl", "render_pkg_aliases")
 load(":requirements_files_by_platform.bzl", "requirements_files_by_platform")
 
@@ -70,7 +71,27 @@ package(default_visibility = ["//visibility:public"])
 exports_files(["requirements.bzl"])
 """
 
+def _evaluate_markers(rctx, requirements, logger = None):
+    python_interpreter = _get_python_interpreter_attr(rctx)
+    stdout = pypi_repo_utils.execute_checked_stdout(
+        rctx,
+        op = "GetPythonVersionForMarkerEval",
+        python = python_interpreter,
+        arguments = [
+            # Run the interpreter in isolated mode, this options implies -E, -P and -s.
+            # Ensures environment variables are ignored that are set in userspace, such as PYTHONPATH,
+            # which may interfere with this invocation.
+            "-I",
+            "-c",
+            "import sys; print(f'{sys.version_info[0]}.{sys.version_info[1]}.{sys.version_info[2]}', end='')",
+        ],
+        srcs = [],
+        logger = logger,
+    )
+    return evaluate_markers(requirements, python_version = stdout)
+
 def _pip_repository_impl(rctx):
+    logger = repo_utils.logger(rctx)
     requirements_by_platform = parse_requirements(
         rctx,
         requirements_by_platform = requirements_files_by_platform(
@@ -82,7 +103,7 @@ def _pip_repository_impl(rctx):
             extra_pip_args = rctx.attr.extra_pip_args,
         ),
         extra_pip_args = rctx.attr.extra_pip_args,
-        evaluate_markers = evaluate_markers,
+        evaluate_markers = lambda requirements: _evaluate_markers(rctx, requirements, logger),
     )
     selected_requirements = {}
     options = None