From e64d623eb2e8f139607bec4bbd6b3e470562e658 Mon Sep 17 00:00:00 2001 From: svenrdz Date: Tue, 17 Sep 2024 15:03:03 +0200 Subject: [PATCH] Fix non utf8 json (#51) * fix(search): correctly parse non UTF-8 response * fix(search): log each exception from an ExceptionGroup --- esgpull/context.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/esgpull/context.py b/esgpull/context.py index 5b55449..cfae14b 100644 --- a/esgpull/context.py +++ b/esgpull/context.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import sys from collections.abc import AsyncIterator, Callable, Coroutine, Sequence from dataclasses import dataclass, field @@ -441,7 +442,9 @@ async def _fetch_one(self, result: RT) -> RT: try: resp = await self.client.send(result.request) resp.raise_for_status() - result.json = resp.json() + result.json = json.loads( + resp.content.decode(encoding="latin-1") + ) logger.info(f"✓ Fetched in {resp.elapsed}s {resp.url}") except HTTPError as exc: result.exc = exc @@ -464,6 +467,8 @@ async def _fetch(self, *in_results: RT) -> AsyncIterator[RT]: group = BaseExceptionGroup("fetch", excs) if self.noraise: logger.exception(group) + for exc in excs: + logger.exception(exc) else: raise group