Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
24f3d88
DRAFT feat(gapic-generator): add support for resumable uploads
parthea Sep 11, 2026
f6c1af6
update goldens
parthea Sep 14, 2026
9d777b4
temporarily pin google-api-core
parthea Sep 16, 2026
1374d07
fix build
parthea Sep 16, 2026
d422468
fix build
parthea Sep 16, 2026
f1e8c9d
fix build
parthea Sep 16, 2026
dddb46a
enable showcase testing
parthea Sep 16, 2026
b8aeb69
enable showcase testing
parthea Sep 16, 2026
ff8d2ea
fix build
parthea Sep 16, 2026
f43cb71
fix build
parthea Sep 17, 2026
1fd0110
attempt to resolve error failed to fetch commit
parthea Sep 17, 2026
13aa272
attempt to resolve error failed to fetch commit
parthea Sep 17, 2026
3835faa
attempt to resolve error failed to fetch commit
parthea Sep 17, 2026
9f3120e
bump showcase version
parthea Sep 17, 2026
4918f15
fix build
parthea Sep 17, 2026
4864acf
restore tests
parthea Sep 17, 2026
fee2bd5
improve testing
parthea Sep 17, 2026
f5c8b4a
add generated code
parthea Sep 21, 2026
38c59dc
add generated sample code
parthea Sep 21, 2026
04fa586
update goldens
parthea Sep 21, 2026
dc468a1
attempt to fix docs build
parthea Sep 21, 2026
159f47b
Remove obsolete code
parthea Sep 21, 2026
b05f894
address review feedback
parthea Sep 22, 2026
c1fe880
address review feedback
parthea Sep 23, 2026
459827f
fix build
parthea Sep 23, 2026
eb3fde4
address review feedback
parthea Sep 23, 2026
c891b99
mypy
parthea Sep 23, 2026
0515ffd
revert
parthea Sep 24, 2026
0719584
attempt to fix docs build
parthea Sep 24, 2026
75af1db
fix build
parthea Sep 24, 2026
cd97a70
update goldens
parthea Sep 24, 2026
ad03f37
attempt to fix build
parthea Sep 24, 2026
08452e9
attempt to fix build
parthea Sep 24, 2026
66ae10d
unpin types-protobuf
parthea Sep 24, 2026
b1b8d9d
fix build
parthea Sep 24, 2026
919cd0e
comment out tests
parthea Sep 24, 2026
a53027a
unpin types-protobuf
parthea Sep 24, 2026
92fa2c2
revert
parthea Sep 24, 2026
8e9acb1
Revert "revert"
parthea Sep 24, 2026
3354d3d
cover
parthea Sep 24, 2026
7539d1d
fix build
parthea Sep 24, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
8 changes: 6 additions & 2 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ name: docs
permissions:
contents: read

env:
NOX_DEFAULT_VENV_BACKEND: "uv"
UV_VENV_SEED: "1"

jobs:
docs:
runs-on: ubuntu-latest
Expand All @@ -34,7 +38,7 @@ jobs:
- name: Install nox
run: |
python -m pip install --upgrade setuptools pip wheel
python -m pip install nox
python -m pip install nox uv
- name: Run docs
env:
BUILD_TYPE: presubmit
Expand Down Expand Up @@ -63,7 +67,7 @@ jobs:
- name: Install nox
run: |
python -m pip install --upgrade setuptools pip wheel
python -m pip install nox
python -m pip install nox uv
- name: Run docfx
env:
BUILD_TYPE: presubmit
Expand Down
55 changes: 54 additions & 1 deletion .github/workflows/gapic-generator-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,10 @@ concurrency:
cancel-in-progress: true

env:
SHOWCASE_VERSION: 0.35.0
SHOWCASE_VERSION: 0.44.0
PROTOC_VERSION: 33.5
LATEST_STABLE_PYTHON: 3.14
OLDEST_PYTHON: '3.10'
PRERELEASE_PYTHON: 3.15
ALL_PYTHON: "['3.10', '3.11', '3.12', '3.13', '3.14', '3.15']"
TRIMMED_PYTHON: "['3.10', '3.14', '3.15']"
Expand Down Expand Up @@ -50,6 +51,7 @@ jobs:
outputs:
all_python: ${{ env.ALL_PYTHON }}
latest_stable_python: ${{ env.LATEST_STABLE_PYTHON }}
oldest_python: ${{ env.OLDEST_PYTHON }}
prerelease_python: ${{ env.PRERELEASE_PYTHON }}
trimmed_python: ${{ env.TRIMMED_PYTHON }}
steps:
Expand Down Expand Up @@ -94,6 +96,57 @@ jobs:
cd packages/gapic-generator
nox -s showcase_unit${{ matrix.variant }}-${MATRIX_PYTHON}

showcase:
needs: python_config
strategy:
# Run showcase tests on the lowest and highest supported runtimes
matrix:
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2121) Remove `showcase_w_rest_async` target when async rest is GA.
python: ["${{ needs.python_config.outputs.oldest_python }}", "${{ needs.python_config.outputs.latest_stable_python }}"]
target: [showcase, showcase_w_rest_async]
logging_scope: ["", "google"]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: Set up Python "${{ matrix.python }}"
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
with:
python-version: "${{ matrix.python }}"
cache: 'pip'
- name: Install system dependencies.
run: |
sudo apt-get update
sudo apt-get install -y curl pandoc unzip
- name: Install showcase
run: |
sudo mkdir -p /usr/src/showcase
sudo chown -R ${USER} /usr/src/
curl --location https://github.com/googleapis/gapic-showcase/releases/download/v${SHOWCASE_VERSION}/gapic-showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz --output /usr/src/showcase/showcase-${SHOWCASE_VERSION}-linux-amd64.tar.gz
cd /usr/src/showcase/
tar -xf showcase-*
./gapic-showcase run &
cd -
- name: Install nox.
run: python -m pip install nox
- name: Install protoc
run: |
sudo mkdir -p /usr/src/protoc/
sudo chown -R ${USER} /usr/src/
curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc-${PROTOC_VERSION}.zip
cd /usr/src/protoc/
unzip protoc-${PROTOC_VERSION}.zip
sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
- name: Run showcase tests.
env:
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2286): Construct nox sessions with logging enabled.
GOOGLE_SDK_PYTHON_LOGGING_SCOPE: ${{ matrix.logging_scope }}
MATRIX_PYTHON: ${{ matrix.python }}
run: |
cd packages/gapic-generator
nox -s ${{ matrix.target }}-${MATRIX_PYTHON}

showcase-mypy:
needs: python_config
runs-on: ubuntu-latest
Expand Down
1 change: 1 addition & 0 deletions packages/gapic-generator/DEVELOPMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,5 +52,6 @@ Execute unit tests by running one of the sessions prefixed with `unit-`.
bazel run //tests/integration:logging_internal_update
bazel run //tests/integration:redis_update
bazel run //tests/integration:redis_selective_update
bazel run //tests/integration:showcase_update
```

9 changes: 9 additions & 0 deletions packages/gapic-generator/WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ http_archive(
urls = ["https://github.com/googleapis/googleapis/archive/{0}.zip".format(_googleapis_sha256)],
)

# Load com_google_gapic_showcase used for testing
_gapic_showcase_sha256 = "5158967e8a41c751b15776142d190428ddb7e99f"
http_archive(
name = "com_google_gapic_showcase",
build_file = "//tests/integration:showcase.BUILD.bazel",
strip_prefix = "gapic-showcase-{}".format(_gapic_showcase_sha256),
urls = ["https://github.com/googleapis/gapic-showcase/archive/{}.zip".format(_gapic_showcase_sha256)],
)

# Load rules_gapic v1.0.0. We patch rules_gapic's .bzl files to inject the
# explicit load for Starlark ProtoInfo from @rules_proto to maintain symbol
# resolution under rules_proto / Protobuf 35.1.
Expand Down
2 changes: 2 additions & 0 deletions packages/gapic-generator/gapic/generator/generator.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,12 +352,14 @@ def _render_template(
and not api_schema.all_library_settings[
api_schema.naming.proto_package
].python_settings.experimental_features.rest_async_io_enabled
and not service.has_resumable_upload_methods
)
or (
"rest_asyncio" in template_name
and not api_schema.all_library_settings[
api_schema.naming.proto_package
].python_settings.experimental_features.rest_async_io_enabled
and not service.has_resumable_upload_methods
)
or ("rest_base" in template_name and "rest" not in opts.transport)
):
Expand Down
17 changes: 12 additions & 5 deletions packages/gapic-generator/gapic/samplegen/samplegen.py
Original file line number Diff line number Diff line change
Expand Up @@ -1183,14 +1183,20 @@ def _get_sample_imports(sample: Dict, rpc: wrappers.Method) -> List[str]:
module_name = sample["module_name"]
module_import = f"from {module_namespace} import {module_name}"

imports = [module_import]
address = rpc.input.meta.address
# This checks if the request message is part of the service proto package.
# If not, we should try to include a separate import statement.
if address.proto_package.startswith(address.api_naming.proto_package):
return [module_import]
else:
request_import = str(address.python_import)
return sorted([module_import, request_import])
if not address.proto_package.startswith(address.api_naming.proto_package):
imports.append(str(address.python_import))

if rpc.is_resumable_upload:
imports.append(
"from google.api_core.resumable_transfer import ResumableUploadConfig, UploadProgress"
)
imports.append("import io")

return sorted(imports)


def generate_sample(
Expand Down Expand Up @@ -1222,6 +1228,7 @@ def generate_sample(

calling_form = types.CallingForm.method_default(rpc)
sample["is_internal"] = rpc.is_internal
sample["is_resumable_upload"] = rpc.is_resumable_upload

v = Validator(rpc, api_schema)
# Tweak some small aspects of the sample to set defaults for optional
Expand Down
8 changes: 8 additions & 0 deletions packages/gapic-generator/gapic/schema/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -1065,6 +1065,13 @@ def has_operations_mixin(self) -> bool:
> 0
)

@cached_property
def has_resumable_upload_methods(self) -> bool:
return any(
service.has_resumable_upload_methods
for service in self.services.values()
)

@cached_property
def _has_iam_overrides(self) -> bool:
if not self.has_iam_mixin:
Expand Down Expand Up @@ -1651,6 +1658,7 @@ def _get_methods(
output=self.api_messages[meth_pb.output_type.lstrip(".")],
retry=retry,
timeout=timeout,
resumable_upload_prefix=self.opts.resumable_upload_prefix,
)

# Done; return the answer.
Expand Down
4 changes: 3 additions & 1 deletion packages/gapic-generator/gapic/schema/imp.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@ def __str__(self) -> str:
# Determine if we need to suppress type checking for this import.
# We do this for protobuf generated files (_pb2) and api_core
# internals where type information might be missing or incomplete.
needs_type_ignore = self.module.endswith("_pb2") or "api_core" in self.package
needs_type_ignore = self.module.endswith("_pb2") or (
"api_core" in self.package and self.module != "resumable_transfer"
)
if needs_type_ignore:
# Use 'import absolute.path as module' syntax to prevent Ruff/isort
# from combining this with other imports. This ensures the
Expand Down
35 changes: 34 additions & 1 deletion packages/gapic-generator/gapic/schema/wrappers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1499,6 +1499,7 @@ class Method:
meta: metadata.Metadata = dataclasses.field(
default_factory=metadata.Metadata,
)
resumable_upload_prefix: str = "resumable/upload"

def __getattr__(self, name):
return getattr(self.method_pb, name)
Expand Down Expand Up @@ -1638,6 +1639,25 @@ def _client_output(self, enable_asyncio: bool):
)
)

if self.is_resumable_upload:
return PythonType(
meta=metadata.Metadata(
address=metadata.Address(
name=(
"AsyncResumableUploadSession"
if enable_asyncio
else "ResumableUploadSession"
),
module="resumable_transfer",
package=("google", "api_core"),
collisions=self.input.ident.collisions,
),
documentation=utils.doc(
"An object representing a resumable upload session."
),
),
)

# Return the usual output.
return self.output

Expand Down Expand Up @@ -1728,6 +1748,11 @@ def http_opt(self) -> Optional[Dict[str, str]]:
# TODO(yon-mg): enums for http verbs?
return answer

@property
def is_resumable_upload(self) -> bool:
"""Return True if this method is a resumable upload method."""
return self.name in ("UploadMedia", "CreateYouTubeVideoUpload")

@property
def path_params(self) -> Sequence[str]:
"""Return the path parameters found in the http annotation path template"""
Expand Down Expand Up @@ -1934,6 +1959,9 @@ def _ref_types(self, recursive: bool) -> Sequence[Union[MessageType, EnumType]]:
if self.paged_result_field and self.paged_result_field.message:
answer.append(self.paged_result_field.message)

if self.is_resumable_upload:
answer.append(self.output)

# Done; return the answer.
return tuple(answer)

Expand Down Expand Up @@ -2208,6 +2236,11 @@ def has_pagers(self) -> bool:
"""Return whether the service has paged methods."""
return any(m.paged_result_field for m in self.methods.values())

@property
def has_resumable_upload_methods(self) -> bool:
"""Return whether the service has resumable upload methods."""
return any(m.is_resumable_upload for m in self.methods.values())

@property
def host(self) -> str:
"""Return the hostname for this service, if specified.
Expand Down Expand Up @@ -2241,7 +2274,7 @@ def shortname(self) -> str:
# Get the shortname from the host
# Real APIs are expected to have format:
# "{api_shortname}.googleapis.com"
return self.host.split(".")[0]
return self.host.split(".")[0].split(":")[0]

@property
def oauth_scopes(self) -> Sequence[str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@
requests: Optional[Iterator[{{ method.input.ident }}]] = None,
*,
{% endif %}
{% if method.is_resumable_upload %}
config: Optional[ResumableUploadConfig] = None,
{% endif %}
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Union[float, object] = gapic_v1.method.DEFAULT,
{{ shared_macros.client_method_metadata_argument()|indent(8) }} = {{ shared_macros.client_method_metadata_default_value() }},
Expand Down Expand Up @@ -65,6 +68,10 @@
The request object iterator.{{ " " }}
{{- method.input.meta.doc|rst(width=72, indent=16, nl=False) }}
{% endif %}
{% if method.is_resumable_upload %}
config (Optional[google.api_core.resumable_transfer.ResumableUploadConfig]):
Optional configuration for the resumable upload session.
{% endif %}
retry (google.api_core.retry.Retry): Designation of what errors, if any,
should be retried.
timeout (float): The timeout for this request.
Expand Down Expand Up @@ -183,6 +190,13 @@
timeout=timeout,
metadata=metadata,
)
{% elif method.is_resumable_upload %}

# Wrap the response in a resumable upload session.
response = {{ method.client_output.ident }}(
config=config,
response_type={{ method.output.ident }},
)
{% elif method.extended_lro and full_extended_lro %}
{{ extended_operation_service_setup(api, method) }}
{% endif %}
Expand Down
Loading
Loading