Skip to content

Commit

Permalink
Fix non utf8 json (#51)
Browse files Browse the repository at this point in the history
* fix(search): correctly parse non UTF-8 response

* fix(search): log each exception from an ExceptionGroup
  • Loading branch information
svenrdz authored Sep 17, 2024
1 parent d48ab70 commit e64d623
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion esgpull/context.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import asyncio
import json
import sys
from collections.abc import AsyncIterator, Callable, Coroutine, Sequence
from dataclasses import dataclass, field
Expand Down Expand Up @@ -441,7 +442,9 @@ async def _fetch_one(self, result: RT) -> RT:
try:
resp = await self.client.send(result.request)
resp.raise_for_status()
result.json = resp.json()
result.json = json.loads(
resp.content.decode(encoding="latin-1")
)
logger.info(f"✓ Fetched in {resp.elapsed}s {resp.url}")
except HTTPError as exc:
result.exc = exc
Expand All @@ -464,6 +467,8 @@ async def _fetch(self, *in_results: RT) -> AsyncIterator[RT]:
group = BaseExceptionGroup("fetch", excs)
if self.noraise:
logger.exception(group)
for exc in excs:
logger.exception(exc)
else:
raise group

Expand Down

0 comments on commit e64d623

Please sign in to comment.