HTTP/2 Stuck around 200 objects?

[william-silversmith]

Hi all, thanks for answering my previous question! I got an experimental setup somewhat working but now I've run into a problem where a download of 512 files stalls at about 200 objects. I increased the timeout to 60 seconds. Here's some code:

    async def httpx_get_raw(client, path, headers):
      raw = b''
      async with client.stream("GET", path, headers=headers) as resp:
        async for raw_bytes in resp.aiter_raw():
          raw += raw_bytes
      return resp, raw

    results = []
    async def async_download(client, path):
      path, start, end = path_to_byte_range(path)
      fullpath = self.abspath(path)
      error = None
      content = None
      encoding = None
      server_hash = None
      server_hash_type = None

      headers = {}
      if start is not None or end is not None:
        start = int(start) if start is not None else ''
        end = int(end - 1) if end is not None else ''
        headers = { f"Range": "bytes={start}-{end}" }

      try:
        response, content = await httpx_get_raw(client, fullpath, headers)
        response.raise_for_status()

        encoding = response.headers.get("content-encoding", None)
        server_hash = None
        server_hash_type = None

        if not raw:
          content = compression.decompress(content, encoding, filename=path)
      except Exception as err:
        error = err

      if raise_errors and error:
        raise error

      results.append({ 
        'path': path, 
        'content': content, 
        'byte_range': (start, end),
        'error': error,
        'compress': encoding,
        'raw': raw,
      })

    async def download():
      async with httpx.AsyncClient(http2=True, timeout=60.0) as client:
        async with trio.open_nursery() as nursery:
          for path in paths:
            nursery.start_soon(async_download, client, path)

    trio.run(download)

EDIT: I should mention that the download is slower but doesn't get stuck if http2 is switched off.