Merge pull request #141 from aryanA101a/useragent

set a user-agent for handle_user_provided_file

Author: benoit74

src/zimscraperlib/constants.py

@@ -10,6 +10,8 @@
 ROOT_DIR = pathlib.Path(__file__).parent
 NAME = pathlib.Path(__file__).parent.name
 SCRAPER = f"{NAME} {__version__}"
+CONTACT = "[email protected]"
+DEFAULT_USER_AGENT = f"{NAME}/{__version__} ({CONTACT})"
 
 UTF8 = "UTF-8"
 

src/zimscraperlib/inputs.py

@@ -9,6 +9,7 @@
 from typing import Optional, Union
 
 from zimscraperlib import logger
+from zimscraperlib.constants import DEFAULT_USER_AGENT
 from zimscraperlib.constants import (
     MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH,
 )
@@ -23,6 +24,7 @@ def handle_user_provided_file(
     dest: Optional[pathlib.Path] = None,
     in_dir: Optional[pathlib.Path] = None,
     nocopy: bool = False,  # noqa: FBT001, FBT002
+    user_agent: Optional[str] = DEFAULT_USER_AGENT,
 ) -> Union[pathlib.Path, None]:
     """path to downloaded or copied a user provided file (URL or path)
 
@@ -44,7 +46,8 @@ def handle_user_provided_file(
 
     if str(source).startswith("http"):
         logger.debug(f"download {source} -> {dest}")
-        stream_file(url=str(source), fpath=dest)
+        headers = {"User-Agent": user_agent} if user_agent else None
+        stream_file(url=str(source), fpath=dest, headers=headers)
     else:
         source = pathlib.Path(source).expanduser().resolve()
         if not source.exists():

tests/conftest.py

@@ -120,6 +120,11 @@ def webp_image():
     return file_src("ninja.webp")
 
 
+@pytest.fixture(scope="module")
+def valid_user_agent():
+    return "name/version (contact)"
+
+
 @pytest.fixture(scope="session")
 def small_zim_file(tmpdir_factory):
     from zimscraperlib.download import stream_file

tests/inputs/test_inputs.py

@@ -6,12 +6,17 @@
 
 import pytest
 
+import zimscraperlib
+from zimscraperlib.constants import CONTACT
 from zimscraperlib.constants import (
     MAXIMUM_DESCRIPTION_METADATA_LENGTH as MAX_DESC_LENGTH,
 )
 from zimscraperlib.constants import (
     MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH,
 )
+from zimscraperlib.constants import (
+    NAME as PROJECT_NAME,
+)
 from zimscraperlib.inputs import compute_descriptions, handle_user_provided_file
 
 
@@ -80,6 +85,54 @@ def test_remote_indir(tmp_path, valid_http_url):
     assert fpath.parent == tmp_path
 
 
+def test_remote_default_user_agent(valid_http_url, monkeypatch):
+    def mock_stream_file(**kwargs):
+        headers = kwargs.get("headers")
+        assert headers is not None
+        user_agent = headers.get("User-Agent")
+        assert isinstance(user_agent, str)
+        assert user_agent.startswith(f"{PROJECT_NAME}/")
+        assert user_agent.endswith(f"({CONTACT})")
+
+    monkeypatch.setattr(
+        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue]
+        "stream_file",
+        mock_stream_file,
+        raising=True,
+    )
+    handle_user_provided_file(source=valid_http_url)
+
+
+def test_remote_provided_user_agent(valid_http_url, valid_user_agent, monkeypatch):
+    def mock_stream_file(**kwargs):
+        headers = kwargs.get("headers")
+        assert headers is not None
+        user_agent = headers.get("User-Agent")
+        assert isinstance(user_agent, str)
+        assert user_agent == valid_user_agent
+
+    monkeypatch.setattr(
+        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue]
+        "stream_file",
+        mock_stream_file,
+        raising=True,
+    )
+    handle_user_provided_file(source=valid_http_url, user_agent=valid_user_agent)
+
+
+def test_remote_provided_none_user_agent(valid_http_url, monkeypatch):
+    def mock_stream_file(**kwargs):
+        assert kwargs.get("headers") is None
+
+    monkeypatch.setattr(
+        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue]
+        "stream_file",
+        mock_stream_file,
+        raising=True,
+    )
+    handle_user_provided_file(source=valid_http_url, user_agent=None)
+
+
 TEXT_NOT_USED = "text not used"
 
 LONG_TEXT = (