Add unit tests for scrapy.utils and scrapy.pipelines (#171)

Author: vdusek

src/apify/scrapy/utils.py

@@ -47,6 +47,9 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict:
         scrapy_request: The Scrapy request to be converted.
         spider: The Scrapy spider that the request is associated with.
 
+    Raises:
+        TypeError: If the scrapy_request is not an instance of the scrapy.Request class.
+
     Returns:
         The converted Apify request.
     """
@@ -88,6 +91,10 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
         apify_request: The Apify request to be converted.
         spider: The Scrapy spider that the request is associated with.
 
+    Raises:
+        TypeError: If the apify_request is not a dictionary.
+        ValueError: If the apify_request does not contain the required keys.
+
     Returns:
         The converted Scrapy request.
     """
@@ -98,7 +105,7 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
     missing_keys = [key for key in required_keys if key not in apify_request]
 
     if missing_keys:
-        raise ValueError(f"apify_request must contain {', '.join(map(repr, missing_keys))} key(s)")
+        raise ValueError(f'apify_request must contain {", ".join(map(repr, missing_keys))} key(s)')
 
     call_id = crypto_random_object_id(8)
     Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')

tests/unit/scrapy/pipelines/__init__.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+from scrapy import Field, Item, Spider
+
+from apify import Actor
+from apify.scrapy.pipelines import ActorDatasetPushPipeline
+
+
+class DummySpider(Spider):
+    name = 'dummy_spider'
+
+
+class DummyItem(Item):
+    a = Field()
+    b = Field()
+    c = Field()
+
+
+class TitleItem(Item):
+    url = Field()
+    title = Field()
+
+
+@pytest.fixture()
+def spider() -> DummySpider:
+    """Fixture to create a "dummy" Scrapy spider."""
+    return DummySpider()
+
+
+@pytest.fixture()
+def pipeline() -> ActorDatasetPushPipeline:
+    """Fixture to create an Actor dataset push pipeline."""
+    return ActorDatasetPushPipeline()
+
+
+@dataclass(frozen=True)
+class TestCase:
+    item: Item
+    item_dict: dict
+    expected_exception: type[Exception] | None
+
+
+test_cases = [
+    TestCase(
+        item=DummyItem(a='string', b=123, c=False),
+        item_dict={'a': 'string', 'b': 123, 'c': False},
+        expected_exception=None,
+    ),
+    TestCase(
+        item=TitleItem(url='https://example.com', title='Example'),
+        item_dict={'url': 'https://example.com', 'title': 'Example'},
+        expected_exception=None,
+    ),
+    TestCase(
+        item=None,
+        item_dict={},
+        expected_exception=TypeError,
+    ),
+]
+
+
+@pytest.mark.parametrize('tc', test_cases)
+async def test__process_item(
+    monkeypatch: pytest.MonkeyPatch,
+    pipeline: ActorDatasetPushPipeline,
+    spider: Spider,
+    tc: TestCase,
+) -> None:
+    dataset = []
+
+    async def mock_push_data(item: dict) -> None:
+        dataset.append(item)
+
+    monkeypatch.setattr(Actor, 'push_data', mock_push_data)
+
+    if tc.expected_exception:
+        with pytest.raises(tc.expected_exception):
+            await pipeline.process_item(tc.item, spider)
+
+    else:
+        output = await pipeline.process_item(tc.item, spider)
+        assert output == tc.item
+        assert dataset == [tc.item_dict]

tests/unit/scrapy/pipelines/test_actor_dataset_push.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+
+from apify.scrapy import get_basic_auth_header
+
+
+@dataclass(frozen=True)
+class TestCase:
+    username: str
+    password: str
+    expected_auth_header: bytes
+
+
+test_cases = [
+    TestCase('username', 'password', b'Basic dXNlcm5hbWU6cGFzc3dvcmQ='),
+    TestCase('john_smith', 'secret_password_123', b'Basic am9obl9zbWl0aDpzZWNyZXRfcGFzc3dvcmRfMTIz'),
+]
+
+
+@pytest.mark.parametrize('tc', test_cases)
+def test__get_basic_auth_header(tc: TestCase) -> None:
+    auth_header = get_basic_auth_header(tc.username, tc.password)
+    assert auth_header == tc.expected_auth_header

tests/unit/scrapy/test_utils.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+from scrapy import Request, Spider
+
+from apify.scrapy.utils import to_apify_request
+
+
+class DummySpider(Spider):
+    name = 'dummy_spider'
+
+
+@pytest.fixture()
+def spider() -> DummySpider:
+    """Fixture to create a "dummy" Scrapy spider."""
+    return DummySpider()
+
+
+@dataclass(frozen=True)
+class TestCase:
+    scrapy_request: Request
+    expected_apify_request: dict | None
+    expected_exception: type[Exception] | None
+
+
+test_cases = [
+    # Valid Scrapy request with 'apify_request_id' and 'apify_request_unique_key'
+    TestCase(
+        scrapy_request=Request(
+            url='https://example.com',
+            method='GET',
+            meta={'apify_request_id': 'abc123', 'apify_request_unique_key': 'https://example.com'},
+        ),
+        expected_apify_request={
+            'url': 'https://example.com',
+            'method': 'GET',
+            'id': 'abc123',
+            'uniqueKey': 'https://example.com',
+            'userData': {'scrapy_request': 'gANjCg...'},  # Example base64-encoded pickle data
+        },
+        expected_exception=None,
+    ),
+    # Valid Scrapy request without 'apify_request_id' and 'apify_request_unique_key'
+    TestCase(
+        scrapy_request=Request(url='https://apify.com', method='GET'),
+        expected_apify_request={
+            'url': 'https://apify.com',
+            'method': 'GET',
+            'userData': {'scrapy_request': 'fhSnfa...'},  # Example base64-encoded pickle data
+        },
+        expected_exception=None,
+    ),
+    # Invalid Scrapy request (not an instance of scrapy.Request)
+    TestCase(
+        scrapy_request=Spider(name='invalid_request'),  # Not a valid Scrapy request
+        expected_apify_request=None,
+        expected_exception=TypeError,
+    ),
+]
+
+
+@pytest.mark.parametrize('tc', test_cases)
+def test__to_apify_request(spider: Spider, tc: TestCase) -> None:
+    if tc.expected_exception:
+        with pytest.raises(tc.expected_exception):
+            to_apify_request(tc.scrapy_request, spider)
+
+    else:
+        apify_request = to_apify_request(tc.scrapy_request, spider)
+        assert isinstance(apify_request, dict)
+        assert tc.expected_apify_request is not None
+        assert apify_request.get('url') == tc.expected_apify_request.get('url')
+        assert apify_request.get('method') == tc.expected_apify_request.get('method')

tests/unit/scrapy/utils/__init__.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import pytest
+from scrapy import Request, Spider
+
+from apify.scrapy.utils import to_scrapy_request
+
+
+class DummySpider(Spider):
+    name = 'dummy_spider'
+
+
+@pytest.fixture()
+def spider() -> DummySpider:
+    """Fixture to create a "dummy" Scrapy spider."""
+    return DummySpider()
+
+
+@dataclass(frozen=True)
+class TestCase:
+    apify_request: dict
+    expected_scrapy_request: Request | None
+    expected_exception: type[Exception] | None
+
+
+test_cases = [
+    # Valid Apify request without 'userData' (directly from Request Queue)
+    TestCase(
+        apify_request={'url': 'https://apify.com/', 'method': 'GET', 'uniqueKey': 'https://apify.com/', 'id': 'fvwscO2UJLdr10B'},
+        expected_scrapy_request=Request(
+            url='https://apify.com/',
+            method='GET',
+            meta={'apify_request_id': 'fvwscO2UJLdr10B', 'apify_request_unique_key': 'https://apify.com/'},
+        ),
+        expected_exception=None,
+    ),
+    # Valid Apify request with 'userData' (reconstruction from encoded Scrapy request)
+    TestCase(
+        apify_request={
+            'url': 'https://apify.com',
+            'method': 'GET',
+            'id': 'fvwscO2UJLdr10B',
+            'uniqueKey': 'https://apify.com',
+            'userData': {
+                'scrapy_request': 'gASVJgIAAAAAAAB9lCiMA3VybJSMEWh0dHBzOi8vYXBpZnkuY29tlIwIY2FsbGJhY2uUTowHZXJy\nYmFja5ROjAdoZWFkZXJzlH2UKEMGQWNjZXB0lF2UQz90ZXh0L2h0bWwsYXBwbGljYXRpb24veGh0\nbWwreG1sLGFwcGxpY2F0aW9uL3htbDtxPTAuOSwqLyo7cT0wLjiUYUMPQWNjZXB0LUxhbmd1YWdl\nlF2UQwJlbpRhQwpVc2VyLUFnZW50lF2UQyNTY3JhcHkvMi4xMS4wICgraHR0cHM6Ly9zY3JhcHku\nb3JnKZRhQw9BY2NlcHQtRW5jb2RpbmeUXZRDDWd6aXAsIGRlZmxhdGWUYXWMBm1ldGhvZJSMA0dF\nVJSMBGJvZHmUQwCUjAdjb29raWVzlH2UjARtZXRhlH2UKIwQYXBpZnlfcmVxdWVzdF9pZJSMD2Z2\nd3NjTzJVSkxkcjEwQpSMGGFwaWZ5X3JlcXVlc3RfdW5pcXVlX2tleZSMEWh0dHBzOi8vYXBpZnku\nY29tlIwQZG93bmxvYWRfdGltZW91dJRHQGaAAAAAAACMDWRvd25sb2FkX3Nsb3SUjAlhcGlmeS5j\nb22UjBBkb3dubG9hZF9sYXRlbmN5lEc/tYIIAAAAAHWMCGVuY29kaW5nlIwFdXRmLTiUjAhwcmlv\ncml0eZRLAIwLZG9udF9maWx0ZXKUiYwFZmxhZ3OUXZSMCWNiX2t3YXJnc5R9lHUu\n',  # noqa: E501
+            },
+        },
+        expected_scrapy_request=Request(
+            url='https://apify.com',
+            method='GET',
+            meta={'apify_request_id': 'fvwscO2UJLdr10B', 'apify_request_unique_key': 'https://apify.com'},
+        ),
+        expected_exception=None,
+    ),
+    # Invalid Apify request (missing 'url' key)
+    TestCase(
+        apify_request={'method': 'GET', 'id': 'invalid123', 'uniqueKey': 'https://invalid.com'},
+        expected_scrapy_request=None,
+        expected_exception=ValueError,
+    ),
+    # Invalid Apify request (missing 'id' key)
+    TestCase(
+        apify_request={'url': 'https://example.com', 'method': 'GET', 'uniqueKey': 'invalid123'},
+        expected_scrapy_request=None,
+        expected_exception=ValueError,
+    ),
+    # Invalid Apify request (non-string 'userData.scrapy_request')
+    TestCase(
+        apify_request={
+            'url': 'https://example.com',
+            'method': 'GET',
+            'id': 'invalid123',
+            'uniqueKey': 'https://example.com',
+            'userData': {'scrapy_request': 123},
+        },
+        expected_scrapy_request=None,
+        expected_exception=TypeError,
+    ),
+]
+
+
+@pytest.mark.parametrize('tc', test_cases)
+def test__to_scrapy_request(spider: Spider, tc: TestCase) -> None:
+    if tc.expected_exception:
+        with pytest.raises(tc.expected_exception):
+            to_scrapy_request(tc.apify_request, spider)
+
+    else:
+        scrapy_request = to_scrapy_request(tc.apify_request, spider)
+
+        assert isinstance(scrapy_request, Request)
+        assert tc.expected_scrapy_request is not None
+        assert scrapy_request.url == tc.expected_scrapy_request.url
+        assert scrapy_request.method == tc.expected_scrapy_request.method
+
+        # Check meta fields
+        assert scrapy_request.meta.get('apify_request_id') == tc.expected_scrapy_request.meta.get('apify_request_id')
+        assert scrapy_request.meta.get('apify_request_unique_key') == tc.expected_scrapy_request.meta.get('apify_request_unique_key')
+
+        # Check if meta field is updated properly when apify_request comes from Scrapy
+        if 'userData' in tc.apify_request and 'scrapy_request' in tc.apify_request['userData']:
+            assert scrapy_request.meta['apify_request_id'] == tc.apify_request['id']
+            assert scrapy_request.meta['apify_request_unique_key'] == tc.apify_request['uniqueKey']