🔖 Release 3.4.3 (#68)

**Fixed**
- Accessing a lazy response (multiplexed enabled) that have multiple
redirects did not work appropriately.

**Changed**
- Response `iter_content` and `iter_line` read chunks as they arrive by
default. The default chunk size is now `-1`. `-1` mean to instruct that
the chunks can be of variable sizes, depending on how packets arrives.
It improves overall performances in streaming content download.
- urllib3.future lower bound constraint has been raised to version
2.4.904 in order to accept `-1` as a chunk size.

HISTORY.md

@@ -1,6 +1,18 @@
 Release History
 ===============
 
+3.4.3 (2024-01-16)
+------------------
+
+**Fixed**
+- Accessing a lazy response (multiplexed enabled) that have multiple redirects did not work appropriately.
+
+**Changed**
+- Response `iter_content` and `iter_line` read chunks as they arrive by default. The default chunk size is now `-1`.
+  `-1` mean to instruct that the chunks can be of variable sizes, depending on how packets arrives. It improves
+  overall performances.
+- urllib3.future lower bound constraint has been raised to version 2.4.904 in order to accept `-1` as a chunk size.
+
 3.4.2 (2024-01-11)
 ------------------
 

docs/user/advanced.rst

@@ -352,7 +352,7 @@ urllib3 :class:`urllib3.HTTPResponse <urllib3.response.HTTPResponse>` at
 :attr:`Response.raw <niquests.Response.raw>`.
 
 If you set ``stream`` to ``True`` when making a request, Niquests cannot
-release the connection back to the pool unless you consume all the data or call
+release the connection back to the pool unless you consume all the data (HTTP/1.1 only) or call
 :meth:`Response.close <niquests.Response.close>`. This can lead to
 inefficiency with connections. If you find yourself partially reading request
 bodies (or not reading them at all) while using ``stream=True``, you should
@@ -1358,10 +1358,14 @@ Setting the source network adapter
 In a complex scenario, you could face the following: "I have multiple network adapters, some can access this and other that.."
 Since Niquests 3.4+, you can configure that aspect per ``Session`` instance.
 
-Having a session without IPv6 enabled should be done that way::
+Having a session that explicitly bind to "10.10.4.1" on port 4444 should be done that way::
 
     import niquests
 
-    session = niquests.Session(source_address=(10.10.4.1, 4444))
+    session = niquests.Session(source_address=("10.10.4.1", 4444))
 
 It will be passed down the the lower stack. No effort required.
+
+.. note:: You can set **0** instead of 4444 to select a random port.
+
+.. note:: You can set **0.0.0.0** to select the network adapter automatically instead, if you wish to set the port only.

pyproject.toml

@@ -41,17 +41,17 @@ dynamic = ["version"]
 dependencies = [
     "charset_normalizer>=2,<4",
     "idna>=2.5,<4",
-    "urllib3.future>=2.4.901,<3",
+    "urllib3.future>=2.4.904,<3",
     "wassima>=1.0.1,<2",
     "kiss_headers>=2,<4",
 ]
 
 [project.optional-dependencies]
 socks = [
-    "urllib3.future[socks]>=2.4.901,<3",
+    "urllib3.future[socks]",
 ]
 http3 = [
-    "urllib3.future[qh3]>=2.4.901,<3"
+    "urllib3.future[qh3]"
 ]
 ocsp = [
     "cryptography<42.0.0,>=41.0.0"

src/niquests/__version__.py

@@ -9,9 +9,9 @@
 __url__: str = "https://niquests.readthedocs.io"
 
 __version__: str
-__version__ = "3.4.2"
+__version__ = "3.4.3"
 
-__build__: int = 0x030402
+__build__: int = 0x030403
 __author__: str = "Kenneth Reitz"
 __author_email__: str = "[email protected]"
 __license__: str = "Apache-2.0"

src/niquests/_async.py

@@ -724,7 +724,7 @@ def __aenter__(self) -> AsyncResponse:
         return self
 
     async def __aiter__(self) -> typing.AsyncIterator[bytes]:
-        async for chunk in await self.iter_content(128):
+        async for chunk in await self.iter_content(ITER_CHUNK_SIZE):
             yield chunk
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
@@ -743,7 +743,7 @@ async def iter_content(
         ...
 
     async def iter_content(  # type: ignore[override]
-        self, chunk_size: int = 1, decode_unicode: bool = False
+        self, chunk_size: int = ITER_CHUNK_SIZE, decode_unicode: bool = False
     ) -> typing.AsyncGenerator[bytes | str, None]:
         async def generate() -> (
             typing.AsyncGenerator[

src/niquests/adapters.py

@@ -12,6 +12,7 @@
 import sys
 import time
 import typing
+from copy import deepcopy
 from datetime import timedelta
 from http.cookiejar import CookieJar
 from threading import RLock
@@ -803,7 +804,9 @@ def send(
 
         return r
 
-    def _future_handler(self, response: Response, low_resp: BaseHTTPResponse) -> None:
+    def _future_handler(
+        self, response: Response, low_resp: BaseHTTPResponse
+    ) -> Response | None:
         stream = typing.cast(
             bool, response._promise.get_parameter("niquests_is_stream")
         )
@@ -826,8 +829,10 @@ def _future_handler(self, response: Response, low_resp: BaseHTTPResponse) -> Non
             response._promise.get_parameter("niquests_kwargs"),
         )
 
-        # mark response as "not lazy" anymore by removing ref to "this"/gather.
-        del response._gather
+        # This mark the response as no longer "lazy"
+        response.raw = low_resp
+        response_promise = response._promise
+        del response._promise
 
         req = response.request
         assert req is not None
@@ -844,7 +849,6 @@ def _future_handler(self, response: Response, low_resp: BaseHTTPResponse) -> Non
 
         # Set encoding.
         response.encoding = get_encoding_from_headers(response.headers)
-        response.raw = low_resp
         response.reason = response.raw.reason
 
         if isinstance(req.url, bytes):
@@ -858,13 +862,10 @@ def _future_handler(self, response: Response, low_resp: BaseHTTPResponse) -> Non
 
         promise_ctx_backup = {
             k: v
-            for k, v in response._promise._parameters.items()
+            for k, v in response_promise._parameters.items()
             if k.startswith("niquests_")
         }
 
-        response_promise = response._promise
-        del response._promise
-
         if allow_redirects:
             next_request = response._resolve_redirect(response, req)
             redirect_count += 1
@@ -875,6 +876,7 @@ def _future_handler(self, response: Response, low_resp: BaseHTTPResponse) -> Non
                 )
 
             if next_request:
+                del self._promises[response_promise.uid]
 
                 def on_post_connection(conn_info: ConnectionInfo) -> None:
                     """This function will be called by urllib3.future just after establishing the connection."""
@@ -904,7 +906,7 @@ def on_post_connection(conn_info: ConnectionInfo) -> None:
 
                 next_promise = self.send(next_request, **kwargs)
 
-                next_promise._gather = lambda: self.gather(response)  # type: ignore[arg-type]
+                next_request.conn_info = deepcopy(next_request.conn_info)
                 next_promise._resolve_redirect = response._resolve_redirect
 
                 if "niquests_origin_response" not in promise_ctx_backup:
@@ -920,14 +922,11 @@ def on_post_connection(conn_info: ConnectionInfo) -> None:
                 for k, v in promise_ctx_backup.items():
                     next_promise._promise.set_parameter(k, v)
 
-                del self._promises[response_promise.uid]
-
-                return
+                return next_promise
         else:
             response._next = response._resolve_redirect(response, req)  # type: ignore[assignment]
 
         del response._resolve_redirect
-
         # In case we handled redirects in a multiplexed connection, we shall reorder history
         # and do a swap.
         if "niquests_origin_response" in promise_ctx_backup:
@@ -984,6 +983,8 @@ def on_post_connection(conn_info: ConnectionInfo) -> None:
 
         del self._promises[response_promise.uid]
 
+        return None
+
     def gather(self, *responses: Response, max_fetch: int | None = None) -> None:
         with self._promise_lock:
             if not self._promises:
@@ -1016,12 +1017,12 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None:
                     )
 
                     if response is None:
-                        raise MultiplexingError(
-                            "Underlying library yield an unexpected response that did not match any of sent request by us"
-                        )
+                        continue
 
                     self._future_handler(response, low_resp)
             else:
+                still_redirects = []
+
                 # ...Or we have a list on which we should focus.
                 for response in responses:
                     if max_fetch is not None and max_fetch == 0:
@@ -1043,18 +1044,25 @@ def gather(self, *responses: Response, max_fetch: int | None = None) -> None:
 
                     if low_resp is None:
                         raise MultiplexingError(
-                            "Underlying library did not recognize our promise when asked to retrieve it. Did you close the session too early?"
+                            "Underlying library did not recognize our promise when asked to retrieve it. "
+                            "Did you close the session too early?"
                         )
 
                     if max_fetch is not None:
                         max_fetch -= 1
 
-                    self._future_handler(response, low_resp)
+                    next_resp = self._future_handler(response, low_resp)
 
-        self._promise_lock.acquire()
+                    if next_resp:
+                        still_redirects.append(next_resp)
 
-        if self._promises:
-            self._promise_lock.release()
-            self.gather()
-        else:
-            self._promise_lock.release()
+                if still_redirects:
+                    self.gather(*still_redirects)
+
+                return
+
+        with self._promise_lock:
+            if not self._promise_lock:
+                return
+
+        self.gather()

src/niquests/models.py

@@ -9,6 +9,7 @@
 import codecs
 import datetime
 import warnings
+import idna
 
 # Import encoding now, to avoid implicit import later.
 # Implicit import within threads may cause LookupError when standard library is in a ZIP,
@@ -119,7 +120,7 @@
 
 DEFAULT_REDIRECT_LIMIT = 30
 CONTENT_CHUNK_SIZE = 10 * 1024
-ITER_CHUNK_SIZE = 512
+ITER_CHUNK_SIZE = -1
 
 
 class TransferProgress:
@@ -354,16 +355,6 @@ def prepare_method(self, method: HttpMethodType | None) -> None:
         """Prepares the given HTTP method."""
         self.method = method.upper() if method else method
 
-    @staticmethod
-    def _get_idna_encoded_host(host: str) -> str:
-        import idna
-
-        try:
-            host = idna.encode(host, uts46=True).decode("utf-8")
-        except idna.IDNAError:
-            raise UnicodeError
-        return host
-
     def prepare_url(self, url: str | None, params: QueryParameterType | None) -> None:
         """Prepares the given HTTP URL."""
         assert url is not None, "Missing URL in PreparedRequest"
@@ -409,8 +400,8 @@ def prepare_url(self, url: str | None, params: QueryParameterType | None) -> Non
         # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
         if not host.isascii():
             try:
-                host = self._get_idna_encoded_host(host)
-            except UnicodeError:
+                host = idna.encode(host, uts46=True).decode("utf-8")
+            except idna.IDNAError:
                 raise InvalidURL("URL has an invalid label.")
         elif host.startswith(("*", ".")):
             raise InvalidURL("URL has an invalid label.")
@@ -923,7 +914,6 @@ class Response:
     #: internals used for lazy responses. Do not try to access those unless you know what you are doing.
     #: they don't always exist.
     _promise: ResponsePromise
-    _gather: typing.Callable[[], None]
     _resolve_redirect: typing.Callable[
         [Response, PreparedRequest], PreparedRequest | None
     ]
@@ -981,7 +971,12 @@ def lazy(self) -> bool:
         Determine if response isn't received and is actually a placeholder.
         Only significant if request was sent through a multiplexed connection.
         """
-        return hasattr(self, "raw") and self.raw is None and hasattr(self, "_gather")
+        return hasattr(self, "raw") and self.raw is None and hasattr(self, "_promise")
+
+    def _gather(self) -> None:
+        """internals used for lazy responses. Do not try to access this unless you know what you are doing."""
+        if hasattr(self, "_promise") and hasattr(self, "connection"):
+            self.connection.gather(self)
 
     def __getattribute__(self, item):
         if item in Response.__lazy_attrs__ and self.lazy:
@@ -1046,7 +1041,7 @@ def __bool__(self) -> bool:
 
     def __iter__(self) -> typing.Generator[bytes, None, None]:
         """Allows you to use a response as an iterator."""
-        return self.iter_content(128)  # type: ignore[return-value]
+        return self.iter_content(ITER_CHUNK_SIZE)  # type: ignore[return-value]
 
     @property
     def ok(self) -> bool:
@@ -1110,7 +1105,7 @@ def iter_content(
         ...
 
     def iter_content(
-        self, chunk_size: int = 1, decode_unicode: bool = False
+        self, chunk_size: int = ITER_CHUNK_SIZE, decode_unicode: bool = False
     ) -> typing.Generator[bytes | str, None, None]:
         """Iterates over the response data.  When stream=True is set on the
         request, this avoids reading the content at once into memory for

src/niquests/sessions.py

@@ -1133,7 +1133,6 @@ def handle_upload_progress(
 
         # We are leveraging a multiplexed connection
         if r.raw is None:
-            r._gather = lambda: adapter.gather(r)
             r._resolve_redirect = lambda x, y: next(
                 self.resolve_redirects(x, y, yield_requests=True, **kwargs),  # type: ignore
                 None,

tests/test_async.py

@@ -56,7 +56,21 @@ async def test_awaitable_get(self):
 
             assert resp.lazy is True
             await s.gather()
+            assert resp.status_code == 200
+
+    async def test_awaitable_redirect_with_lazy(self):
+        async with AsyncSession(multiplexed=True) as s:
+            resp = await s.get("https://pie.dev/redirect/3")
+
+            assert resp.lazy is True
+            await s.gather()
+            assert resp.status_code == 200
 
+    async def test_awaitable_redirect_direct_access_with_lazy(self):
+        async with AsyncSession(multiplexed=True) as s:
+            resp = await s.get("https://pie.dev/redirect/3")
+
+            assert resp.lazy is True
             assert resp.status_code == 200
 
     async def test_awaitable_get_direct_access_lazy(self):