Merge branch 'main' into fix/lockfile-inconsistency

Author: aignas

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/python/private,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

.bcr/metadata.template.json

@@ -10,6 +10,11 @@
       "name": "Thulio Ferraz Assis",
       "email": "[email protected]",
       "github": "f0rmiga"
+    },
+    {
+      "name": "Ignas Anikevicius",
+      "email": "[email protected]",
+      "github": "aignas"
     }
   ],
   "repository": [

.gitattributes

@@ -1 +1,2 @@
+python/features.bzl export-subst
 tools/publish/*.txt linguist-generated=true

CHANGELOG.md

@@ -33,14 +33,23 @@ A brief description of the categories of changes:
   ([2310](https://github.com/bazelbuild/rules_python/issues/2310)).
 * (publish) The dependencies have been updated to the latest available versions
   for the `twine` publishing rule.
-- (bzlmod) `pip.parse` spoke repository naming has changed in places where the
+* (bzlmod) `pip.parse` spoke repository naming has changed in places where the
   users specify different package versions per platform in the same hub repository.
   We advise the users to use the `hub` repository directly to avoid such breakage in
   the future.
 
 {#v0-0-0-fixed}
 ### Fixed
-- (bzlmod) The extension evaluation has been adjusted to always generate the
+* (bzlmod) Generate `config_setting` values for all available toolchains instead
+  of only the registered toolchains, which restores the previous behaviour that
+  `bzlmod` users would have observed.
+* (pypi) (Bazel 7.4+) Allow spaces in filenames included in `whl_library`s
+  ([617](https://github.com/bazelbuild/rules_python/issues/617)).
+* (whl_library) Remove `--no-build-isolation` to allow non-hermetic sdist builds
+  by default. Users wishing to keep this argument and to enforce more hermetic
+  builds can do so by passing the argument in
+  [`pip.parse#extra_pip_args`](https://rules-python.readthedocs.io/en/latest/api/rules_python/python/extensions/pip.html#pip.parse.extra_pip_args)
+* (bzlmod) The extension evaluation has been adjusted to always generate the
   same lock file irrespective if `experimental_index_url` is set by any module
   or not. Fixes
   [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
@@ -54,11 +63,12 @@ A brief description of the categories of changes:
   `requirements_linux.txt`, `requirements_windows.txt` for each respective OS
   and one extra file `requirements_universal.txt` if you prefer a single file.
   The `requirements.txt` file may be removed in the future.
-- (bzlmod) `pip.parse` now supports `whl-only` setup using 
+* (bzlmod) `pip.parse` now supports `whl-only` setup using 
   `download_only = True` where users can specify multiple requirements files
   and use the `pip` backend to do the downloading. This was only available for
   users setting {bzl:obj}`pip.parse.experimental_index_url`, but now users have
   more options whilst we continue to work on stabilizing the experimental feature.
+* The rules_python version is now reported in `//python/features.bzl#features.version`
 
 {#v0-0-0-removed}
 ### Removed

DEVELOPING.md

@@ -24,6 +24,8 @@ Start from a clean checkout at `main`.
 Before running through the release it's good to run the build and the tests locally, and make sure CI is passing. You can
 also test-drive the commit in an existing Bazel workspace to sanity check functionality.
 
+### Releasing from HEAD
+
 #### Steps
 1. [Determine the next semantic version number](#determining-semantic-version)
 1. Create a tag and push, e.g. `git tag 0.5.0 upstream/main && git push upstream --tags`
@@ -42,9 +44,31 @@ To find if there were any features added or incompatible changes made, review
 the commit history. This can be done using github by going to the url:
 `https://github.com/bazelbuild/rules_python/compare/<VERSION>...main`.
 
+### Patch release with cherry picks
+
+If a patch release from head would contain changes that aren't appropriate for
+a patch release, then the patch release needs to be based on the original
+release tag and the patch changes cherry-picked into it.
+
+In this example, release `0.37.0` is being patched to create release `0.37.1`.
+The fix being included is commit `deadbeef`.
+
+1. `git checkout -b release/0.37 0.37.0`
+1. `git push upstream release/0.37`
+1. `git cherry-pick -x deadbeef`
+1. Fix merge conflicts, if any. If `MODULE.bazel.lock` conflicts occur, then
+   run `pre-commit run update-bzlmod-lockfiles -a`
+1. `git cherry-pick --continue` (if applicable)
+1. `git push upstream`
+
+If multiple commits need to be applied, repeat the `git cherry-pick` step for
+each.
+
+Once the release branch is in the desired state, use `git tag` to tag it, as
+done with a release from head. Release automation will do the rest.
+
 #### After release creation in Github
 
-1. Ping @philwo to get the new release added to mirror.bazel.build. See [this comment on issue #400](https://github.com/bazelbuild/rules_python/issues/400#issuecomment-779159530) for more context.
 1. Announce the release in the #python channel in the Bazel slack (bazelbuild.slack.com).
 
 ## Secrets

docs/BUILD.bazel

@@ -97,11 +97,11 @@ sphinx_stardocs(
         "//python/cc:py_cc_toolchain_bzl",
         "//python/cc:py_cc_toolchain_info_bzl",
         "//python/entry_points:py_console_script_binary_bzl",
-        "//python/private:py_binary_rule_bazel_bzl",
+        "//python/private:py_binary_rule_bzl",
         "//python/private:py_cc_toolchain_rule_bzl",
-        "//python/private:py_library_rule_bazel_bzl",
+        "//python/private:py_library_rule_bzl",
         "//python/private:py_runtime_rule_bzl",
-        "//python/private:py_test_rule_bazel_bzl",
+        "//python/private:py_test_rule_bzl",
         "//python/private/api:py_common_api_bzl",
     ] + ([
         # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension

docs/api/rules_python/python/config_settings/index.md

@@ -33,6 +33,44 @@ one of the values that `rules_python` maintains.
 Parses the value of the `python_version` and transforms it into a `X.Y` value.
 :::
 
+:::{bzl:target} is_python_*
+config_settings to match Python versions
+
+The name pattern is `is_python_X.Y` (to match major.minor) and `is_python_X.Y.Z`
+(to match major.minor.patch).
+
+Note that the set of available targets depends on the configured
+`TOOL_VERSIONS`. Versions may not always be available if the root module has
+customized them, or as older Python versions are removed from rules_python's set
+of builtin, known versions.
+
+If you need to match a version that isn't present, then you have two options:
+1. Manually define a `config_setting` and have it match {obj}`--python_version`
+   or {ob}`python_version_major_minor`. This works best when you don't control the
+   root module, or don't want to rely on the MODULE.bazel configuration. Such
+   a config settings would look like:
+   ```
+   # Match any 3.5 version
+   config_setting(
+       name = "is_python_3.5",
+       flag_values = {
+           "@rules_python//python/config_settings:python_version_major_minor": "3.5",
+       }
+   )
+   # Match exactly 3.5.1
+   config_setting(
+       name = "is_python_3.5.1",
+       flag_values = {
+           "@rules_python//python/config_settings:python_version": "3.5.1",
+       }
+   )
+   ```
+
+2. Use {obj}`python.single_override` to re-introduce the desired version so
+   that the corresponding `//python/config_setting:is_python_XXX` target is
+   generated.
+:::
+
 ::::{bzl:flag} exec_tools_toolchain
 Determines if the {obj}`exec_tools_toolchain_type` toolchain is enabled.
 

docs/conf.py

@@ -69,9 +69,9 @@
     "api/python/defs": "/api/rules_python/python/defs.html",
     "api/python/index": "/api/rules_python/python/index.html",
     "api/python/py_runtime_info": "/api/rules_python/python/py_runtime_info.html",
-    "api/python/private/common/py_library_rule_bazel": "/api/rules_python/python/private/py_library_rule_bazel.html",
+    "api/python/private/common/py_library_rule_bazel": "/api/rules_python/python/private/py_library_rule.html",
     "api/python/private/common/py_test_rule_bazel": "/api/rules_python/python/private/py_test_rule_bazel.html",
-    "api/python/private/common/py_binary_rule_bazel": "/api/rules_python/python/private/py_binary_rule_bazel.html",
+    "api/python/private/common/py_binary_rule_bazel": "/api/rules_python/python/private/py_binary_rule.html",
     "api/python/private/common/py_runtime_rule": "/api/rules_python/python/private/py_runtime_rule.html",
     "api/python/extensions/pip": "/api/rules_python/python/extensions/pip.html",
     "api/python/extensions/python": "/api/rules_python/python/extensions/python.html",

examples/bzlmod/MODULE.bazel

@@ -40,14 +40,16 @@ python.toolchain(
 # One can override the actual toolchain versions that are available, which can be useful
 # when optimizing what gets downloaded and when.
 python.override(
-    available_python_versions = [
-        "3.10.9",
-        "3.9.18",
-        "3.9.19",
-        # The following is used by the `other_module` and we need to include it here
-        # as well.
-        "3.11.8",
-    ],
+    # NOTE: These are disabled in the example because transitive dependencies
+    # require versions not listed here.
+    # available_python_versions = [
+    #     "3.10.9",
+    #     "3.9.18",
+    #     "3.9.19",
+    #     # The following is used by the `other_module` and we need to include it here
+    #     # as well.
+    #     "3.11.8",
+    # ],
     # Also override the `minor_mapping` so that the root module,
     # instead of rules_python's defaulting to the latest available version,
     # controls what full version is used when `3.x` is requested.

examples/bzlmod/MODULE.bazel.lock

@@ -30,6 +30,7 @@ python.toolchain(
 )
 use_repo(
     python,
+    "pythons_hub",
     python = "python_versions",
 )
 

examples/multi_python_versions/MODULE.bazel

@@ -1,9 +1,14 @@
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
+load("@bazel_skylib//rules:diff_test.bzl", "diff_test")
+load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@python//3.10:defs.bzl", py_binary_3_10 = "py_binary", py_test_3_10 = "py_test")
 load("@python//3.11:defs.bzl", py_binary_3_11 = "py_binary", py_test_3_11 = "py_test")
 load("@python//3.8:defs.bzl", py_binary_3_8 = "py_binary", py_test_3_8 = "py_test")
 load("@python//3.9:defs.bzl", py_binary_3_9 = "py_binary", py_test_3_9 = "py_test")
+load("@pythons_hub//:versions.bzl", "MINOR_MAPPING", "PYTHON_VERSIONS")
 load("@rules_python//python:defs.bzl", "py_binary", "py_test")
+load("@rules_python//python:versions.bzl", DEFAULT_MINOR_MAPPING = "MINOR_MAPPING", DEFAULT_TOOL_VERSIONS = "TOOL_VERSIONS")
+load("@rules_python//python/private:text_util.bzl", "render")  # buildifier: disable=bzl-visibility
 load("@rules_shell//shell:sh_test.bzl", "sh_test")
 
 copy_file(
@@ -183,3 +188,37 @@ sh_test(
         "VERSION_PY_BINARY": "$(rootpath :version_3_10)",
     },
 )
+
+# The following test ensures that default toolchain versions are the same as in
+# the TOOL_VERSIONS array.
+
+# NOTE @aignas 2024-10-26: This test here is to do a sanity check and not
+# include extra dependencies - if rules_testing is included here, we can
+# potentially uses `rules_testing` for a more lightweight test.
+write_file(
+    name = "default_python_versions",
+    out = "default_python_versions.txt",
+    content = [
+        "MINOR_MAPPING:",
+        render.dict(dict(sorted(DEFAULT_MINOR_MAPPING.items()))),
+        "PYTHON_VERSIONS:",
+        render.list(sorted(DEFAULT_TOOL_VERSIONS)),
+    ],
+)
+
+write_file(
+    name = "pythons_hub_versions",
+    out = "pythons_hub_versions.txt",
+    content = [
+        "MINOR_MAPPING:",
+        render.dict(dict(sorted(MINOR_MAPPING.items()))),
+        "PYTHON_VERSIONS:",
+        render.list(sorted(PYTHON_VERSIONS)),
+    ],
+)
+
+diff_test(
+    name = "test_versions",
+    file1 = "default_python_versions",
+    file2 = "pythons_hub_versions",
+)