fix: update correct requirements lock file when using os specific lock files

Author: Rasrack

.bazelci/presubmit.yml

@@ -454,6 +454,54 @@ tasks:
     - "! git diff --exit-code"
     - "bazel run //:requirements.update"
     - "git diff --exit-code"
+  integration_test_compile_pip_os_specific_requirements_ubuntu:
+    <<: *reusable_build_test_all
+    name: compile_pip_os_specific_requirements integration tests on Ubuntu
+    working_directory: tests/compile_pip_requirements
+    platform: ubuntu2004
+    shell_commands:
+    # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the
+    # right thing.
+    - "echo '' > requirements_lock_linux.txt"
+    - "! git diff --exit-code"
+    - "bazel run //:os_specific_requirements.update"
+    - "git diff --exit-code"
+  integration_test_compile_pip_os_specific_requirements_debian:
+    <<: *reusable_build_test_all
+    name: compile_pip_os_specific_requirements integration tests on Debian
+    working_directory: tests/compile_pip_requirements
+    platform: debian11
+    shell_commands:
+    # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the
+    # right thing.
+    - "echo '' > requirements_lock_linux.txt"
+    - "! git diff --exit-code"
+    - "bazel run //:os_specific_requirements.update"
+    - "git diff --exit-code"
+  integration_test_compile_pip_os_specific_requirements_macos:
+    <<: *reusable_build_test_all
+    name: compile_pip_os_specific_requirements integration tests on macOS
+    working_directory: tests/compile_pip_requirements
+    platform: macos
+    shell_commands:
+    # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the
+    # right thing.
+    - "echo '' > requirements_lock_darwin.txt"
+    - "! git diff --exit-code"
+    - "bazel run //:os_specific_requirements.update"
+    - "git diff --exit-code"
+  integration_test_compile_pip_os_specific_requirements_windows:
+    <<: *reusable_build_test_all
+    name: compile_pip_os_specific_requirements integration tests on Windows
+    working_directory: tests/compile_pip_requirements
+    platform: windows
+    shell_commands:
+    # Make a change to the locked requirements and then assert that //:os_specific_requirements.update does the
+    # right thing.
+    - "echo '' > requirements_lock_windows.txt"
+    - "! git diff --exit-code"
+    - "bazel run //:os_specific_requirements.update"
+    - "git diff --exit-code"
 
   integration_test_pip_repository_entry_points_ubuntu_min:
     <<: *minimum_supported_version

python/pip_install/tools/dependency_resolver/dependency_resolver.py

@@ -88,6 +88,10 @@ def _select_golden_requirements_file(
     requirements_in_path = Path(requirements_in)
     resolved_requirements_in = str(requirements_in_path.resolve())
 
+    requirements_file = _select_golden_requirements_file(
+        requirements_txt, requirements_linux, requirements_darwin, requirements_windows
+    )
+
     # Before loading click, set the locale for its parser.
     # If it leaks through to the system setting, it may fail:
     # RuntimeError: Click will abort further execution because Python 3 was configured to use ASCII
@@ -108,11 +112,11 @@ def _select_golden_requirements_file(
         sys.argv.append(os.environ["TEST_TMPDIR"])
         # Make a copy for pip-compile to read and mutate.
         requirements_out = os.path.join(
-            os.environ["TEST_TMPDIR"], os.path.basename(requirements_txt) + ".out"
+            os.environ["TEST_TMPDIR"], os.path.basename(requirements_file) + ".out"
         )
         # Those two files won't necessarily be on the same filesystem, so we can't use os.replace
         # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link.
-        shutil.copy(requirements_txt, requirements_out)
+        shutil.copy(requirements_file, requirements_out)
 
     update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % (
         update_target_label,
@@ -123,28 +127,28 @@ def _select_golden_requirements_file(
 
     sys.argv.append("--generate-hashes")
     sys.argv.append("--output-file")
-    sys.argv.append(requirements_txt if UPDATE else requirements_out)
+    sys.argv.append(requirements_file if UPDATE else requirements_out)
     sys.argv.append(
         requirements_in if requirements_in_path.exists() else resolved_requirements_in
     )
 
     if UPDATE:
-        print("Updating " + requirements_txt)
+        print("Updating " + requirements_file)
         if "BUILD_WORKSPACE_DIRECTORY" in os.environ:
             workspace = os.environ["BUILD_WORKSPACE_DIRECTORY"]
-            requirements_txt_tree = os.path.join(workspace, requirements_txt)
-            # In most cases, requirements_txt will be a symlink to the real file in the source tree.
-            # If symlinks are not enabled (e.g. on Windows), then requirements_txt will be a copy,
+            requirements_file_tree = os.path.join(workspace, requirements_file)
+            # In most cases, requirements_file will be a symlink to the real file in the source tree.
+            # If symlinks are not enabled (e.g. on Windows), then requirements_file will be a copy,
             # and we should copy the updated requirements back to the source tree.
-            if not os.path.samefile(requirements_txt, requirements_txt_tree):
+            if not os.path.samefile(requirements_file, requirements_file_tree):
                 atexit.register(
-                    lambda: shutil.copy(requirements_txt, requirements_txt_tree)
+                    lambda: shutil.copy(requirements_file, requirements_file_tree)
                 )
         cli()
     else:
         # cli will exit(0) on success
         try:
-            print("Checking " + requirements_txt)
+            print("Checking " + requirements_file)
             cli()
             print("cli() should exit", file=sys.stderr)
             sys.exit(1)
@@ -158,13 +162,7 @@ def _select_golden_requirements_file(
                 )
                 sys.exit(1)
             elif e.code == 0:
-                golden_filename = _select_golden_requirements_file(
-                    requirements_txt,
-                    requirements_linux,
-                    requirements_darwin,
-                    requirements_windows,
-                )
-                golden = open(golden_filename).readlines()
+                golden = open(requirements_file).readlines()
                 out = open(requirements_out).readlines()
                 if golden != out:
                     import difflib

tests/compile_pip_requirements/BUILD.bazel

@@ -32,3 +32,33 @@ compile_pip_requirements(
     requirements_in = "requirements.txt",
     requirements_txt = "requirements_lock.txt",
 )
+
+genrule(
+    name = "generate_os_specific_requirements_in",
+    srcs = [],
+    outs = ["requirements_os_specific.in"],
+    cmd = """
+cat > $@ <<EOF
+pip==22.3.0 ; sys_platform == "linux"
+pip==22.2.2 ; sys_platform == "darwin"
+pip==22.2.1 ; sys_platform == "win32"
+EOF
+""",
+)
+
+compile_pip_requirements(
+    name = "requirements_os_specific",
+    data = [
+        "requirements_extra.in",
+        "requirements_os_specific.in",
+    ],
+    extra_args = [
+        "--allow-unsafe",
+        "--resolver=backtracking",
+    ],
+    requirements_darwin = "requirements_lock_darwin.txt",
+    requirements_in = "requirements_os_specific.in",
+    requirements_linux = "requirements_lock_linux.txt",
+    requirements_txt = "requirements_lock.txt",
+    requirements_windows = "requirements_lock_windows.txt",
+)

tests/compile_pip_requirements/requirements_lock_darwin.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    bazel run //:requirements_os_specific.update
+#
+pip==22.2.2 ; sys_platform == "darwin" \
+    --hash=sha256:3fd1929db052f056d7a998439176d3333fa1b3f6c1ad881de1885c0717608a4b \
+    --hash=sha256:b61a374b5bc40a6e982426aede40c9b5a08ff20e640f5b56977f4f91fed1e39a
+    # via -r ./requirements_os_specific.in

tests/compile_pip_requirements/requirements_lock_linux.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    bazel run //:requirements_os_specific.update
+#
+pip==22.3 ; sys_platform == "linux" \
+    --hash=sha256:1daab4b8d3b97d1d763caeb01a4640a2250a0ea899e257b1e44b9eded91e15ab \
+    --hash=sha256:8182aec21dad6c0a49a2a3d121a87cd524b950e0b6092b181625f07ebdde7530
+    # via -r ./requirements_os_specific.in

tests/compile_pip_requirements/requirements_lock_windows.txt

@@ -0,0 +1,10 @@
+#
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
+#
+#    bazel run //:requirements_os_specific.update
+#
+pip==22.2.1 ; sys_platform == "win32" \
+    --hash=sha256:0bbbc87dfbe6eed217beff0021f8b7dea04c8f4a0baa9d31dc4cff281ffc5b2b \
+    --hash=sha256:50516e47a2b79e77446f0d05649f0d53772c192571486236b1905492bfc24bac
+    # via -r ./requirements_os_specific.in