feat: Compress requests using brotli algo

- apify/apify-core#28971 added support for brotli compression to BE.
- Pros: higher compression, cons: more CPU intensive.
- The brotli dependencies are defined as optional, one has to explicitly enable it and choose one. If none is available, the code falls back to gzip.
- JS client doesn't compress requests that are too small, this Python client compresses just everything. No change done, I only noticed and stating it.

Author: mixalturek

README.md

@@ -48,6 +48,15 @@
 
     or any other Python package manager that consumes PyPI.
 
+    To enable brotli request-body compression (better than gzip, especially for large payloads), install the optional extra:
+
+    ```bash
+    pip install "apify-client[brotli]"
+    # or
+    uv add "apify-client[brotli]"
+    ```
+
+    Without this extra the client falls back to gzip automatically — no code changes needed.
 
 - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/):
 

pyproject.toml

@@ -30,6 +30,12 @@ dependencies = [
     "pydantic[email]>=2.11.0",
 ]
 
+[project.optional-dependencies]
+brotli = [
+    "brotli>=1.0.9; platform_python_implementation == 'CPython'",
+    "brotlicffi>=1.0.9; platform_python_implementation != 'CPython'",
+]
+
 [project.urls]
 "Apify Homepage" = "https://apify.com"
 "Homepage" = "https://docs.apify.com/api/client/python/"
@@ -203,7 +209,7 @@ exclude = ["website/versioned_docs"]
 unused-ignore-comment = "error"
 
 [[tool.ty.overrides]]
-include = ["docs/**/*.py", "website/**/*.py"]
+include = ["docs/**/*.py", "website/**/*.py", "tests/unit/test_http_clients.py"]
 [tool.ty.overrides.rules]
 unresolved-import = "ignore"
 

src/apify_client/http_clients/_base.py

@@ -23,10 +23,25 @@
 from apify_client._utils import to_seconds
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Iterator, Mapping
+    from collections.abc import AsyncIterator, Callable, Iterator, Mapping
 
     from apify_client.types import JsonSerializable, Timeout
 
+_brotli_compress: Callable[[bytes | bytearray], bytes] | None = None
+
+if not TYPE_CHECKING:
+    try:
+        import brotli
+
+        _brotli_compress = brotli.compress
+    except ImportError:
+        try:
+            import brotlicffi
+
+            _brotli_compress = brotlicffi.compress
+        except ImportError:
+            pass
+
 
 @docs_group('HTTP clients')
 @runtime_checkable
@@ -203,22 +218,30 @@ def _prepare_request_call(
         data: str | bytes | bytearray | None = None,
         json: JsonSerializable | None = None,
     ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]:
-        """Prepare headers, params, and body for an HTTP request. Serializes JSON and applies gzip compression."""
+        """Prepare headers, params, and body for an HTTP request. Serializes JSON and compresses the body.
+
+        Uses brotli compression (`Content-Encoding: br`) when `brotli` or `brotlicffi` is installed,
+        otherwise falls back to gzip (`Content-Encoding: gzip`).
+        """
         if json is not None and data is not None:
             raise ValueError('Cannot pass both "json" and "data" parameters at the same time!')
 
         headers = dict(headers) if headers else {}
 
-        # Dump JSON data to string so it can be gzipped.
+        # Dump JSON data to string so it can be compressed.
         if json is not None:
             data = jsonlib.dumps(json, ensure_ascii=False, allow_nan=False, default=str).encode('utf-8')
             headers['Content-Type'] = 'application/json'
 
         if isinstance(data, (str, bytes, bytearray)):
             if isinstance(data, str):
                 data = data.encode('utf-8')
-            data = gzip.compress(data)
-            headers['Content-Encoding'] = 'gzip'
+            if _brotli_compress is not None:
+                data = _brotli_compress(data)
+                headers['Content-Encoding'] = 'br'
+            else:
+                data = gzip.compress(data)
+                headers['Content-Encoding'] = 'gzip'
 
         return (headers, self._parse_params(params), data)
 

tests/unit/test_http_clients.py

@@ -13,6 +13,29 @@
 from apify_client.http_clients import HttpClient, HttpClientAsync, HttpResponse, ImpitHttpClient, ImpitHttpClientAsync
 from apify_client.http_clients._impit import _is_retryable_error
 
+try:
+    import brotli as _brotli_mod
+
+    _EXPECTED_ENCODING = 'br'
+
+    def _decompress(data: bytes) -> bytes:
+        return _brotli_mod.decompress(data)
+
+except ImportError:
+    try:
+        import brotlicffi as _brotli_mod
+
+        _EXPECTED_ENCODING = 'br'
+
+        def _decompress(data: bytes) -> bytes:
+            return _brotli_mod.decompress(data)
+
+    except ImportError:
+        _EXPECTED_ENCODING = 'gzip'
+
+        def _decompress(data: bytes) -> bytes:
+            return gzip.decompress(data)
+
 
 class _ConcreteHttpClient(HttpClient):
     """Minimal concrete HttpClient for testing base class helpers."""
@@ -278,7 +301,7 @@ def test_prepare_request_call_with_json() -> None:
     headers, _params, data = client._prepare_request_call(json=json_data)
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
 
@@ -290,12 +313,10 @@ def test_prepare_request_call_with_empty_dict_json() -> None:
     headers, _params, data = client._prepare_request_call(json={})
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
-    # Verify the gzipped data contains the JSON
-    decompressed = gzip.decompress(data)
-    assert decompressed == b'{}'
+    assert _decompress(data) == b'{}'
 
 
 def test_prepare_request_call_with_empty_list_json() -> None:
@@ -305,12 +326,10 @@ def test_prepare_request_call_with_empty_list_json() -> None:
     headers, _params, data = client._prepare_request_call(json=[])
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
-    # Verify the gzipped data contains the JSON
-    decompressed = gzip.decompress(data)
-    assert decompressed == b'[]'
+    assert _decompress(data) == b'[]'
 
 
 def test_prepare_request_call_with_zero_json() -> None:
@@ -320,12 +339,10 @@ def test_prepare_request_call_with_zero_json() -> None:
     headers, _params, data = client._prepare_request_call(json=0)
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
-    # Verify the gzipped data contains the JSON
-    decompressed = gzip.decompress(data)
-    assert decompressed == b'0'
+    assert _decompress(data) == b'0'
 
 
 def test_prepare_request_call_with_false_json() -> None:
@@ -335,12 +352,10 @@ def test_prepare_request_call_with_false_json() -> None:
     headers, _params, data = client._prepare_request_call(json=False)
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
-    # Verify the gzipped data contains the JSON
-    decompressed = gzip.decompress(data)
-    assert decompressed == b'false'
+    assert _decompress(data) == b'false'
 
 
 def test_prepare_request_call_with_empty_string_json() -> None:
@@ -350,12 +365,10 @@ def test_prepare_request_call_with_empty_string_json() -> None:
     headers, _params, data = client._prepare_request_call(json='')
 
     assert headers['Content-Type'] == 'application/json'
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert data is not None
     assert isinstance(data, bytes)
-    # Verify the gzipped data contains the JSON
-    decompressed = gzip.decompress(data)
-    assert decompressed == b'""'
+    assert _decompress(data) == b'""'
 
 
 def test_prepare_request_call_with_string_data() -> None:
@@ -364,7 +377,7 @@ def test_prepare_request_call_with_string_data() -> None:
 
     headers, _params, data = client._prepare_request_call(data='test string')
 
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert isinstance(data, bytes)
 
 
@@ -374,7 +387,7 @@ def test_prepare_request_call_with_bytes_data() -> None:
 
     headers, _params, data = client._prepare_request_call(data=b'test bytes')
 
-    assert headers['Content-Encoding'] == 'gzip'
+    assert headers['Content-Encoding'] == _EXPECTED_ENCODING
     assert isinstance(data, bytes)
 
 
@@ -452,3 +465,41 @@ def test_build_url_with_params_mixed() -> None:
     assert 'tags=a' in url
     assert 'tags=b' in url
     assert 'name=test' in url
+
+
+def test_prepare_request_call_brotli_compression(monkeypatch: pytest.MonkeyPatch) -> None:
+    """When a brotli compressor is available, request body uses brotli (Content-Encoding: br)."""
+    brotlicffi = pytest.importorskip('brotlicffi')
+    monkeypatch.setattr('apify_client.http_clients._base._brotli_compress', brotlicffi.compress)
+
+    client = _ConcreteHttpClient()
+    headers, _, data = client._prepare_request_call(json={'k': 'v'})
+
+    assert headers['Content-Encoding'] == 'br'
+    assert data is not None
+    assert brotlicffi.decompress(data) == b'{"k": "v"}'
+
+
+def test_prepare_request_call_brotli_library_compression(monkeypatch: pytest.MonkeyPatch) -> None:
+    """When the brotli (C extension) library is available, request body uses brotli (Content-Encoding: br)."""
+    brotli = pytest.importorskip('brotli')
+    monkeypatch.setattr('apify_client.http_clients._base._brotli_compress', brotli.compress)
+
+    client = _ConcreteHttpClient()
+    headers, _, data = client._prepare_request_call(json={'k': 'v'})
+
+    assert headers['Content-Encoding'] == 'br'
+    assert data is not None
+    assert brotli.decompress(data) == b'{"k": "v"}'
+
+
+def test_prepare_request_call_gzip_fallback_without_brotli(monkeypatch: pytest.MonkeyPatch) -> None:
+    """When no brotli library is available, request body falls back to gzip (Content-Encoding: gzip)."""
+    monkeypatch.setattr('apify_client.http_clients._base._brotli_compress', None)
+
+    client = _ConcreteHttpClient()
+    headers, _, data = client._prepare_request_call(json={'k': 'v'})
+
+    assert headers['Content-Encoding'] == 'gzip'
+    assert data is not None
+    assert gzip.decompress(data) == b'{"k": "v"}'

tests/unit/test_run_charge.py

@@ -4,6 +4,24 @@
 import json
 from typing import TYPE_CHECKING
 
+try:
+    import brotlicffi as _brotli_mod
+
+    def _decompress(data: bytes) -> bytes:
+        return _brotli_mod.decompress(data)
+
+except ImportError:
+    try:
+        import brotli as _brotli_mod  # type: ignore[no-redef]
+
+        def _decompress(data: bytes) -> bytes:
+            return _brotli_mod.decompress(data)
+
+    except ImportError:
+
+        def _decompress(data: bytes) -> bytes:
+            return gzip.decompress(data)
+
 import pytest
 from werkzeug import Request, Response
 
@@ -18,7 +36,10 @@
 
 def _decode_body(request: Request) -> dict:
     raw = request.get_data()
-    if request.headers.get('Content-Encoding') == 'gzip':
+    encoding = request.headers.get('Content-Encoding')
+    if encoding == 'br':
+        raw = _decompress(raw)
+    elif encoding == 'gzip':
         raw = gzip.decompress(raw)
     return json.loads(raw)