Tests for scrapy-puppeteer (#22)

* Added testing of actions

* Added mockserver (not working)

* Only http

* screenshot (not working)

* Working screenshot

* Go back and forward testing

* Refactoring

* Working mockserver!

* Problem with click action...

* Added testing in GitHub Actions

* Deleted pytest.ini

* Pytest.ini example for developers

.github/workflows/python-test.yml

@@ -0,0 +1,32 @@
+name: Test Scrapy-Puppeteer Library
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.7.x"  # Min Python version (No 3.6 version)
+          - python-version: "3.8.x"
+          - python-version: "3.9.x"
+          - python-version: "3.10.x"
+          - python-version: "3.x"  # Last Python version
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set Python version ${{ matrix.python-version }} Up
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install -r requirements.txt
+
+      - name: Run Tests
+        run: |
+          python -m pytest

pytest.ini.example

@@ -0,0 +1,6 @@
+Maybe you want to test new features or minor changes on your local PC.
+In this case you need to provide `pytest.ini` file in the root of the project:
+
+[pytest]
+
+pythonpath = <path to the project>

tests/actions/constants.py

@@ -0,0 +1,23 @@
+from random import randint
+from itertools import combinations
+
+URLS = ("https://some_url.com", "not_url/not_url")
+WAIT_UNTIL = ("load", "domcontentloaded", "networkidle0")
+WAIT_OPTS = [None]
+SELECTORS = ("nothing", "tr.td::attr(something)")
+CLICK_OPTS = [None]
+
+
+def __gen_nav_opts():
+    options = [None]
+    for opt_num in range(1, 5):
+        for comb in combinations(WAIT_UNTIL, opt_num):
+            timeout = randint(0, 100) * 1000
+            options.append({
+                'timeout': timeout,
+                'waitUntil': list(comb),
+            })
+    return options
+
+
+NAV_OPTS = __gen_nav_opts()

tests/actions/test_actions.py

@@ -0,0 +1,78 @@
+from pytest import mark
+from scrapypuppeteer.actions import GoTo, GoForward, GoBack, Click, Scroll
+from itertools import product
+from constants import URLS, NAV_OPTS, WAIT_OPTS, SELECTORS, CLICK_OPTS
+
+
+def _gen_goto():
+    for url, nav_opt, wait_opt in product(URLS, NAV_OPTS, WAIT_OPTS):
+        expected = {
+            'url': url,
+            'navigationOptions': nav_opt,
+            'waitOptions': wait_opt,
+        }
+        yield url, nav_opt, wait_opt, expected
+
+
+def _gen_back_forward():
+    for nav_opt, wait_opt in product(NAV_OPTS, WAIT_OPTS):
+        expected = {
+            'navigationOptions': nav_opt,
+            'waitOptions': wait_opt,
+        }
+        yield nav_opt, wait_opt, expected
+
+
+def _gen_click():
+    for selector, click_opt, nav_opt, wait_opt in product(SELECTORS, CLICK_OPTS, NAV_OPTS, WAIT_OPTS):
+        expected = {
+            'selector': selector,
+            'clickOptions': click_opt,
+            'waitOptions': wait_opt,
+            'navigationOptions': nav_opt,
+        }
+        yield selector, click_opt, nav_opt, wait_opt, expected
+
+
+def _gen_scroll():
+    for selector, wait_opt in product(SELECTORS, WAIT_OPTS):
+        expected = {
+            'selector': selector,
+            'waitOptions': wait_opt
+        }
+        yield selector, wait_opt, expected
+
+
+@mark.parametrize("url, navigation_options, wait_options, expected",
+                  _gen_goto())
+def test_goto(url, navigation_options, wait_options, expected):
+    action = GoTo(url, navigation_options, wait_options)
+    assert action.payload() == expected
+
+
+@mark.parametrize("navigation_options, wait_options, expected",
+                  _gen_back_forward())
+def test_go_forward(navigation_options, wait_options, expected):
+    action = GoForward(navigation_options, wait_options)
+    assert action.payload() == expected
+
+
+@mark.parametrize("navigation_options, wait_options, expected",
+                  _gen_back_forward())
+def test_go_forward(navigation_options, wait_options, expected):
+    action = GoBack(navigation_options, wait_options)
+    assert action.payload() == expected
+
+
+@mark.parametrize("selector, click_options, navigation_options, wait_options, expected",
+                  _gen_click())
+def test_click(selector, click_options, navigation_options, wait_options, expected):
+    action = Click(selector, click_options, wait_options, navigation_options)
+    assert action.payload() == expected
+
+
+@mark.parametrize("selector, wait_options, expected",
+                  _gen_scroll())
+def test_scroll(selector, wait_options, expected):
+    action = Scroll(selector, wait_options)
+    assert action.payload() == expected

tests/middleware/test_middleware.py

@@ -0,0 +1,53 @@
+from tests.spiders import (
+    GoToSpider,
+    GoBackForwardSpider,
+    ClickSpider,
+    ScreenshotSpider,
+    CustomJsActionSpider,
+)
+from tests.mockserver import MockServer
+from twisted.trial.unittest import TestCase
+from twisted.internet import defer
+from scrapy.utils.test import get_crawler
+
+
+class PuppeteerCrawlTest(TestCase):
+    SETTINGS = {
+        'DOWNLOADER_MIDDLEWARES': {
+            'scrapypuppeteer.middleware.PuppeteerServiceDownloaderMiddleware': 1042
+        },
+        'PUPPETEER_SERVICE_URL': None,
+    }
+
+    def setUp(self):
+        self.mockserver = MockServer()
+        self.mockserver.__enter__()
+        self.SETTINGS['PUPPETEER_SERVICE_URL'] = self.mockserver.http_address
+
+    def tearDown(self):
+        self.mockserver.__exit__(None, None, None)
+
+    def _start_testing(self, spider_cls, expected):
+        crawler = get_crawler(spider_cls, self.SETTINGS)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertEqual(len(crawler.spider.urls_visited), expected)
+
+    @defer.inlineCallbacks
+    def test_goto(self):
+        yield from self._start_testing(GoToSpider, 1)
+
+    @defer.inlineCallbacks
+    def test_back_forward(self):
+        yield from self._start_testing(GoBackForwardSpider, 1)
+
+    @defer.inlineCallbacks
+    def test_click(self):
+        yield from self._start_testing(ClickSpider, 1)
+
+    @defer.inlineCallbacks
+    def test_screenshot(self):
+        yield from self._start_testing(ScreenshotSpider, 1)
+
+    @defer.inlineCallbacks
+    def test_custom_js_action(self):
+        yield from self._start_testing(CustomJsActionSpider, 1)

tests/middleware/view.py

@@ -0,0 +1,26 @@
+from scrapy import Request
+
+
+import scrapy
+
+
+class ViewSpider(scrapy.Spider):
+    name = "view"
+
+    start_urls = ["https://www.google.com/recaptcha/api2/demo"]
+
+    custom_settings = {}
+
+    def start_requests(self):
+        for url in self.start_urls:
+            yield Request(url, callback=self.parse, errback=self.errback)
+
+    def parse(self, response, **kwargs):
+        self.log("WE ARE PARSING RESPONSE!")
+        self.log(response)
+        self.log(response.body)
+        self.log("WE HAVE PARSED RESPONSE!")
+
+    def errback(self, failure):
+        self.log("We are in error processing!")
+        self.log(failure)