Fix breakage when built against protobuf head (grpc#34945)

This PR is required in order to upgrade to the `master` branch version of protobuf. To do this, we upgrade to the latest version of `rules_python` by:

- Adding in explicit dependencies on all PyPi Bazel test dependencies
  - Resolving the circular dependency this creates for `xds-protos` by giving it a Bazel build
    - Generating the Python code for `xds-protos` as part of `generate-projects.sh` since doing so directly in Bazel would be highly fragily, dependent on many other projects' Bazel builds

Closes grpc#34945

PiperOrigin-RevId: 599875020

Author: gnossen

WORKSPACE

@@ -59,23 +59,27 @@ local_repository(
     path = "third_party/utf8_range",
 )
 
-load("@io_bazel_rules_python//python:pip.bzl", "pip_install")
+load("@rules_python//python:pip.bzl", "pip_parse")
 
-pip_install(
+pip_parse(
     name = "grpc_python_dependencies",
-    requirements = "@com_github_grpc_grpc//:requirements.bazel.txt",
+    requirements_lock = "@com_github_grpc_grpc//:requirements.bazel.txt",
 )
 
+load("@grpc_python_dependencies//:requirements.bzl", "install_deps")
+
+install_deps()
+
 load("@com_google_protobuf//bazel:system_python.bzl", "system_python")
 
 system_python(
     name = "system_python",
     minimum_python_version = "3.7",
 )
 
-load("@system_python//:pip.bzl", "pip_parse")
+load("@system_python//:pip.bzl", system_pip_parse = "pip_parse")
 
-pip_parse(
+system_pip_parse(
     name = "pip_deps",
     requirements = "@com_google_protobuf//python:requirements.txt",
     requirements_overrides = {

bazel/grpc_extra_deps.bzl

@@ -22,6 +22,7 @@ load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@envoy_api//bazel:repositories.bzl", "api_dependencies")
 load("@google_cloud_cpp//bazel:google_cloud_cpp_deps.bzl", "google_cloud_cpp_deps")
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
+load("@rules_python//python:repositories.bzl", "py_repositories")
 
 def grpc_extra_deps(ignore_version_differences = False):
     """Loads the extra dependencies.
@@ -71,3 +72,5 @@ def grpc_extra_deps(ignore_version_differences = False):
     )
 
     google_cloud_cpp_deps()
+
+    py_repositories()

bazel/grpc_python_deps.bzl

@@ -19,13 +19,12 @@ load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_confi
 # buildifier: disable=unnamed-macro
 def grpc_python_deps():
     """Loads dependencies for gRPC Python."""
-    if "io_bazel_rules_python" not in native.existing_rules():
+    if "rules_python" not in native.existing_rules():
         http_archive(
-            name = "io_bazel_rules_python",
-            url = "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz",
-            sha256 = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea",
-            patches = ["@com_github_grpc_grpc//third_party:rules_python.patch"],
-            patch_args = ["-p1"],
+            name = "rules_python",
+            sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
+            strip_prefix = "rules_python-0.26.0",
+            url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
         )
 
     python_configure(name = "local_config_python")

black.toml

@@ -21,6 +21,8 @@ extend-exclude = '''
   | tools/distrib/python/grpcio_tools/protoc_lib_deps.py
   | .*_pb2.py  # autogenerated Protocol Buffer files
   | .*_pb2_grpc.py  # autogenerated Protocol Buffer gRPC files
+  # AUTO-GENERATED By tools/distrib/python/xds_protos/build.py
+  | tools/distrib/python/xds_protos/.*
 )
 '''
 
@@ -44,6 +46,7 @@ skip_glob = [
     "*pb2*.py",
     "*pb2*.pyi",
     "**/site-packages/**/*",
+    "tools/distrib/python/xds_protos/*",
 ]
 single_line_exclusions = ["typing"]
 force_single_line = true

requirements.bazel.txt

@@ -15,6 +15,25 @@ zope.event==4.5.0
 setuptools==44.1.1
 xds-protos==0.0.11
 absl-py==1.4.0
+google-cloud-trace==1.11.3
+opencensus-context==0.1.3
+google-cloud-monitoring==2.16.0
+google-api-core==2.14.0
+proto-plus==1.22.3
+google-auth==2.23.4
 googleapis-common-protos==1.61.0
+cachetools==5.3.2
+charset-normalizer==3.3.2                                                                         
+pyasn1==0.5.0
+rsa==4.9                                                   
+greenlet==1.1.3.post0
+zope.interface==6.1
 opentelemetry-sdk==1.21.0
 opentelemetry-api==1.21.0
+importlib-metadata==7.0.1
+Deprecated==1.2.14
+opentelemetry-semantic-conventions==0.43b0
+typing-extensions==4.9.0
+pyasn1-modules==0.3.0
+zipp==3.17.0
+wrapt==1.16.0

src/python/grpcio_csds/grpc_csds/BUILD.bazel

@@ -12,16 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-load("@grpc_python_dependencies//:requirements.bzl", "requirement")
-
 package(default_visibility = ["//visibility:public"])
 
 py_library(
     name = "grpc_csds",
     srcs = glob(["*.py"]),
     imports = ["../"],
     deps = [
-        requirement("xds-protos"),
         "//src/python/grpcio/grpc:grpcio",
+        "//tools/distrib/python/xds_protos",
     ],
 )

third_party/BUILD

@@ -11,6 +11,5 @@ exports_files([
     "enum34.BUILD",
     "futures.BUILD",
     "protobuf.patch",
-    "rules_python.patch",
     "protoc-gen-validate.patch",
 ])

third_party/rules_python.patch

@@ -34,6 +34,12 @@ python3 tools/buildgen/extract_metadata_from_bazel_xml.py
 
 tools/buildgen/build_cleaner.py build_handwritten.yaml
 
+# /usr/local/google/home/rbellevi/dev/tmp/grpc/venv/bin/python3: No module named virtualenv
+# Generate xds-protos
+[[ -d generate_projects_virtual_environment ]] || python3 -m virtualenv generate_projects_virtual_environment
+generate_projects_virtual_environment/bin/pip install --upgrade --ignore-installed grpcio-tools==1.59.0
+generate_projects_virtual_environment/bin/python tools/distrib/python/xds_protos/build.py
+
 # check build_autogenerated.yaml is already in its "clean" form
 TEST=true tools/buildgen/build_cleaner.py build_autogenerated.yaml
 

tools/buildgen/generate_projects.sh

@@ -296,6 +296,12 @@ def replace_copyright(license_text, file_text, filename):
         or filename.endswith(".upb_minitable.c")
     ):
         continue
+    # Allow empty __init__.py files for code generated by xds_protos
+    if filename.startswith("tools/distrib/python/xds_protos") and (
+        filename.endswith("__init__.py")
+        or filename.endswith("generated_file_import_test.py")
+    ):
+        continue
     ext = os.path.splitext(filename)[1]
     base = os.path.basename(filename)
     if filename.startswith(_ENFORCE_CPP_STYLE_COMMENT_PATH_PREFIX) and ext in [

tools/distrib/check_copyright.py

@@ -1,11 +1,2 @@
 distrib_virtualenv/
-__init__.py
-generated_file_import_test.py
-envoy/
-udpa/
-validate/
-google/
-opencensus/
-xds/
 build/
-opentelemetry/

tools/distrib/python/.gitignore

@@ -6,3 +6,4 @@ grpc_root/
 *.cpp
 *.egg-info
 *.so
+third_party/

tools/distrib/python/grpcio_tools/.gitignore

@@ -297,7 +297,7 @@
     'third_party/protobuf/src/google/protobuf/unknown_field_set.cc',
     'third_party/protobuf/src/google/protobuf/wire_format.cc',
     'third_party/protobuf/src/google/protobuf/wire_format_lite.cc',
-    'third_party/utf8_range/utf8_validity.cc'
+    'third_party/protobuf/third_party/utf8_range/utf8_validity.cc'
 ]
 
 PROTO_FILES=[
@@ -316,7 +316,9 @@
 ]
 
 CC_INCLUDES=[
- 'third_party/abseil-cpp', 'third_party/protobuf/src', 'third_party/utf8_range'
+    'third_party/abseil-cpp',
+    'third_party/protobuf/src',
+    'third_party/protobuf/third_party/utf8_range'
 ]
 PROTO_INCLUDE='third_party/protobuf/src'
 

tools/distrib/python/grpcio_tools/protoc_lib_deps.py

@@ -63,7 +63,7 @@
 EXTERNAL_LINKS = [
     ("@com_google_absl//", "third_party/abseil-cpp/"),
     ("@com_google_protobuf//", "third_party/protobuf/"),
-    ("@utf8_range//:", "third_party/utf8_range/"),
+    ("@utf8_range//", "third_party/protobuf/third_party/utf8_range"),
 ]
 
 PROTOBUF_PROTO_PREFIX = "@com_google_protobuf//src/"
@@ -72,7 +72,7 @@
 CC_INCLUDES = [
     os.path.join("third_party", "abseil-cpp"),
     os.path.join("third_party", "protobuf", "src"),
-    os.path.join("third_party", "utf8_range"),
+    os.path.join("third_party", "protobuf", "third_party", "utf8_range"),
 ]
 
 # include path for .proto files
@@ -88,7 +88,10 @@
     ("src/compiler", "grpc_root/src/compiler"),
     ("third_party/abseil-cpp/absl", "third_party/abseil-cpp/absl"),
     ("third_party/protobuf/src", "third_party/protobuf/src"),
-    ("third_party/utf8_range", "third_party/utf8_range"),
+    (
+        "third_party/protobuf/third_party/utf8_range",
+        "third_party/protobuf/third_party/utf8_range",
+    ),
 ]
 
 DELETE_TARGETS_ON_CLEANUP = ["third_party"]

tools/distrib/python/make_grpcio_tools.py

@@ -0,0 +1,37 @@
+# Copyright 2024 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//src/python/grpcio_csds:__subpackages__"])
+
+_XDS_PROTO_DIRECTORIES = [
+    "contrib",
+    "envoy",
+    "google",
+    "opencensus",
+    "opentelemetry",
+    "udpa",
+    "validate",
+    "xds",
+]
+
+py_library(
+    name = "xds_protos",
+    srcs = (["__init__.py"] +
+            glob([d + "/**/*.py" for d in _XDS_PROTO_DIRECTORIES])),
+    imports = ["."],
+    deps = [
+        "//src/python/grpcio/grpc:grpcio",
+        "@com_google_protobuf//:protobuf_python",
+    ],
+)

tools/distrib/python/xds_protos/BUILD.bazel

@@ -105,6 +105,7 @@ def has_grpc_service(proto_package_path: str) -> bool:
 
 
 def compile_protos(proto_root: str, sub_dir: str = ".") -> None:
+    compiled_any = False
     for root, _, files in os.walk(os.path.join(proto_root, sub_dir)):
         proto_package_path = os.path.relpath(root, proto_root)
         if proto_package_path in EXCLUDE_PROTO_PACKAGES_LIST:
@@ -113,6 +114,7 @@ def compile_protos(proto_root: str, sub_dir: str = ".") -> None:
         for file_name in files:
             if file_name.endswith(".proto"):
                 # Compile proto
+                compiled_any = True
                 if has_grpc_service(proto_package_path):
                     return_code = protoc.main(
                         COMPILE_BOTH + [os.path.join(root, file_name)]
@@ -127,6 +129,10 @@ def compile_protos(proto_root: str, sub_dir: str = ".") -> None:
                     )
                 if return_code != 0:
                     raise Exception("error: {} failed".format(COMPILE_BOTH))
+    # Ensure a deterministic order.
+    TEST_IMPORTS.sort()
+    if not compiled_any:
+        raise Exception("No proto files found at {}. Did you update git submodules?".format(proto_root, sub_dir))
 
 
 def create_init_file(path: str, package_path: str = "") -> None: