Allow unit and selenium tests to run in parallel (#1611)

Author: istride

.github/workflows/config.yml

@@ -13,20 +13,24 @@ jobs:
 
   django-unit-tests:
     runs-on: ubuntu-latest
+    env:
+      IOGT_TEST_PARALLEL: "2"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: make test
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: django-coverage-report
           path: htmlcov
 
   selenium-tests:
     runs-on: ubuntu-latest
+    env:
+      IOGT_TEST_PARALLEL: "1"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - run: make selenium-test
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         with:
           name: django-coverage-report
           path: htmlcov

Dockerfile.dev

@@ -4,9 +4,10 @@ ENV PYTHONUNBUFFERED 1
 
 WORKDIR /app
 
-RUN apt update -y
-RUN apt install gettext -y
+RUN apt-get update --yes --quiet \
+ && apt-get install --yes --quiet --no-install-recommends gettext tini \
+ && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip
-ADD ./requirements.txt ./requirements.dev.txt ./
-RUN pip install -r requirements.dev.txt
+COPY requirements.txt requirements.dev.txt /
+RUN pip install --no-cache-dir --upgrade pip \
+ && pip install --no-cache-dir -r /requirements.dev.txt

Makefile

@@ -1,3 +1,5 @@
+IOGT_TEST_PARALLEL ?= 1
+
 # Build the project
 build:
 	docker-compose build
@@ -20,15 +22,14 @@ update_elasticsearch_index:
 test:
 	docker-compose -f docker-compose.test.yml up --build -d django
 	docker-compose -f docker-compose.test.yml exec -T django python manage.py collectstatic --noinput
-	docker-compose -f docker-compose.test.yml exec -T django coverage run --source='.' manage.py test --noinput
+	docker-compose -f docker-compose.test.yml exec -T django coverage run --source='.' manage.py test --noinput --parallel $(IOGT_TEST_PARALLEL)
 	docker-compose -f docker-compose.test.yml exec -T django coverage html
 	docker-compose -f docker-compose.test.yml down --remove-orphans
 selenium-test: selenium-up selenium-local selenium-down
 selenium-up:
-	docker-compose -f docker-compose.selenium.yml up --build -d
+	docker-compose -f docker-compose.selenium.yml up --build -d --scale chrome=$(IOGT_TEST_PARALLEL)
 	docker-compose -f docker-compose.selenium.yml exec -T django python manage.py collectstatic --noinput
 selenium-local:
-	docker-compose exec -T django python manage.py test selenium_tests
+	docker-compose -f docker-compose.selenium.yml exec -T django python manage.py test selenium_tests --parallel $(IOGT_TEST_PARALLEL)
 selenium-down:
 	docker-compose -f docker-compose.selenium.yml down --remove-orphans
-

README.md

@@ -129,11 +129,20 @@ docker-compose run django python manage.py autopopulate_main_menus
 
 ## Running Tests
 
-Run the following command:
+Run the following commands:
 ```
-make test
+make tests
+make selenium-test
 ```
 
+In parallel, for example, with 4 processes:
+```
+IOGT_TEST_PARALLEL=4 make test
+IOGT_TEST_PARALLEL=4 make selenium-test
+```
+
+More details of the Selenium tests can be found in the [Selenium test README][9].
+
 ## Configuring the Chatbot
 
 Follow instructions [here](messaging/README.md)
@@ -184,4 +193,5 @@ Follow instructions [here](iogt_content_migration/README.md)
 [5]: https://wagtail.io/
 [6]: https://github.com/wagtail/wagtail/wiki/Release-schedule
 [7]: ./docs/cache.md
-[8]: ./docs/troubleshooting.md
+[8]: ./docs/troubleshooting.md
+[9]: ./selenium_tests/README.md

docker-compose.selenium.yml

@@ -1,50 +1,31 @@
 version: '3.5'
 
 services:
-
   django:
     build:
       context: ./
       dockerfile: Dockerfile.dev
     environment:
-      DJANGO_SETTINGS_MODULE: iogt.settings.docker_compose_dev
-      COMMIT_HASH: asdfghjkl
-      WAGTAILTRANSFER_SECRET_KEY: wagtailtransfer-secret-key
-      WAGTAILTRANSFER_SOURCE_NAME: iogt_global
-      WAGTAILTRANSFER_SOURCE_BASE_URL: https://example.com/wagtail-transfer/
-      WAGTAILTRANSFER_SOURCE_SECRET_KEY: wagtailtransfer-source-secret-key
-      BASE_URL: 'http://localhost:8000'
-    command: python manage.py runserver 0.0.0.0:8000
+      DJANGO_SETTINGS_MODULE: iogt.settings.test
+    command: ["tini", "--", "sleep", "infinity"]
     volumes:
       - ./:/app/
-    ports:
-      - "8000:8000"
     depends_on:
-      - elasticsearch
+      - database
       - selenium-hub
-
-  elasticsearch:
-    image: 'docker.elastic.co/elasticsearch/elasticsearch:7.12.1'
+  database:
+    image: postgres:14-alpine
     environment:
-      - discovery.type=single-node
-    ports:
-      - "9200:9200"
-
+      POSTGRES_PASSWORD: iogt
+  selenium-hub:
+    image: selenium/hub:4.9.1-20230508
   chrome:
-    image: selenium/node-chrome:4.7.2-20221219
+    image: selenium/node-chrome:4.9.1-20230508
     shm_size: 2gb
     depends_on:
       - selenium-hub
     environment:
-      - SE_EVENT_BUS_HOST=selenium-hub
-      - SE_EVENT_BUS_PUBLISH_PORT=4442
-      - SE_EVENT_BUS_SUBSCRIBE_PORT=4443
-    ports:
-      - "5900:5900"
-
-  selenium-hub:
-    image: selenium/hub:4.7.2-20221219
-    ports:
-      - "4442:4442"
-      - "4443:4443"
-      - "4444:4444"
+      SE_EVENT_BUS_HOST: selenium-hub
+      SE_EVENT_BUS_PUBLISH_PORT: "4442"
+      SE_EVENT_BUS_SUBSCRIBE_PORT: "4443"
+      SE_VNC_NO_PASSWORD: "1"

iogt/settings/test.py

@@ -1,18 +1,26 @@
+from os import getenv
+
 from .base import *
 
-SECRET_KEY = "##secret_key_for_testing##"
 
+ALLOWED_HOSTS = ['*']
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql',
-        'NAME': os.environ.get('DB_NAME'),
-        'USER': os.environ.get('DB_USER'),
-        'PASSWORD': os.environ.get('DB_PASSWORD'),
-        'HOST': os.environ.get('DB_HOST'),
-        'PORT': os.environ.get('DB_PORT'),
+        'NAME': getenv('DB_NAME', 'postgres'),
+        'USER': getenv('DB_USER', 'postgres'),
+        'PASSWORD': getenv('DB_PASSWORD', 'iogt'),
+        'HOST': getenv('DB_HOST', 'database'),
+        'PORT': getenv('DB_PORT', '5432'),
     }
 }
-
+SE_APP_HOST = "django"
+SE_HUB_HOST = "selenium-hub"
+SE_HUB_PORT = "4444"
+SECRET_KEY = "##secret_key_for_testing##"
 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
-
-DEBUG = True
+WAGTAILSEARCH_BACKENDS = {
+    'default': {
+        'BACKEND': 'wagtail.search.backends.database',
+    }
+}

requirements.dev.txt

@@ -5,6 +5,6 @@ Faker
 beautifulsoup4
 django-test-plus>=1.4.0
 coverage
-selenium==4.7.2
+selenium==4.9.1
 factory-boy==3.2.*
-wagtail-factories==4.0.*
+wagtail-factories==4.0.*

selenium_tests/README.md

@@ -1,6 +1,6 @@
 # Selenium Tests
 
-We are using Selenium to carry out functional testing in the IoGT project. Specifically, we are using Selenium grid which gives us the option to run tests on multiple browsers.
+We are using Selenium to carry out functional testing in the IoGT project. Specifically, we are using Selenium Grid which gives us the option to run tests on multiple browsers.
 
 The functional tests have been broken down into Categories > Groups > Tests.
 
@@ -13,7 +13,7 @@ For general guidance on how to write Selenium tests, review the existing tests w
 
 The tests are configured to run automatically through GitHub actions, although this may not be practical during intial test development.
 
-The Selenium Grid and WebDriver is run from a Docker container. Therefore, to run tests locally you need to install Docker Desktop and have it running on your computer.
+The Selenium Grid and WebDriver is run from a Docker container. Therefore, to run tests locally you need to install Docker and have it running on your computer.
 
 Then run the following commands:
 ```
@@ -26,7 +26,38 @@ To run the tests themselves, run:
 make selenium-local
 ```
 
-You can view the tests as they run locally using a number of [different methods][1].
+You can view the tests as they run locally via [a couple of different methods][1] - namely VNC and noVNC.
 
+## Running tests in parallel
 
-[1]: https://github.com/SeleniumHQ/docker-selenium#using-a-vnc-client
+Make sure that no ports are being exposed from the Selenium nodes. For example, don't use the 'ports' option to expose the VNC port (5900) because this will prevent two or more Selenium nodes from starting by causing them all try to bind to port 5900 on the host.
+
+Scale up the number of Selenium nodes.
+```
+docker-compose -f docker-compose.selenium.yml up -d --scale chrome=4
+```
+
+Run the tests in parallel.
+```
+docker-compose -f docker-compose.selenium.yml exec -T django python manage.py test selenium_tests --parallel 4
+```
+
+The environment variable `IOGT_TEST_PARALLEL` when running the whole test suite.
+```
+IOGT_TEST_PARALLEL=4 make selenium-test
+```
+
+## Architecture
+
+App service container starts and sleeps - waiting for test executions. A sleeping service is required so that the Selenium nodes can target the test server that is started by the test suite.
+
+PostgreSQL service - used instead of Sqlite for better multi-threaded performance and for being more like a production environment.
+
+Selenium Hub - to route webdriver commands to Selenium nodes
+
+Selenium Nodes - currently only Chrome, but other nodes with different browsers and capabilities could be created e.g. Firefox, Safari, Edge, or with Javascript disabled and/or mobile features enabled.
+
+Once all services are started, test executions can be started on the app service. Each test execution starts a test app server, creates a new test database, then begins running tests. When run in parallel, the main test runner will start several child test runners that each create their own test server and database, and requires a single Selenium node to run tests in.
+
+
+[1]: https://github.com/SeleniumHQ/docker-selenium#debugging

selenium_tests/base.py

@@ -1,34 +1,25 @@
+from django.db import connections
+from django.conf import settings
+from django.core.management import call_command
 from django.test import LiveServerTestCase
 from selenium import webdriver
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
-from wagtail.core.models import Page
-from selenium_tests.pages import BasePage
-from selenium_tests.pages import LoginPage
-from selenium_tests.pages import LogoutPage
-from wagtail.core.models import Site
+from wagtail.core.models import Page, Site
 from wagtail_factories import SiteFactory
+
 from home.factories import HomePageFactory
+from selenium_tests.pages import BasePage, LoginPage, LogoutPage
+
 
 class BaseSeleniumTests(LiveServerTestCase):
 
-    fixtures = ['selenium_tests/locales.json']    
+    fixtures = ['selenium_tests/locales.json']
 
-    host = 'django'
-    port = 9000
+    host = settings.SE_APP_HOST
 
     @classmethod
     def setUpClass(cls):
-        options = webdriver.ChromeOptions()
-        options.add_argument('--ignore-ssl-errors=yes')
-        options.add_argument('--ignore-certificate-errors')
-        options.add_argument("--window-size=480,720")
-        options.add_argument("--start-maximized")
-        cls.selenium = webdriver.Remote(
-            command_executor='http://selenium-hub:4444/wd/hub',
-            desired_capabilities=DesiredCapabilities.CHROME,
-            options=options
-        )
-        cls.selenium.implicitly_wait(10)
+        cls.selenium = create_remote_webdriver()
         super(BaseSeleniumTests, cls).setUpClass()
 
     @classmethod
@@ -62,3 +53,56 @@ def setup_blank_site(self):
             is_default_site=True,
             root_page=self.home
         )
+
+    # https://github.com/wagtail/wagtail/issues/1824#issuecomment-450575883
+    # Identical to TransactionTestCase._fixture_teardown except that 'allow_cascade' is
+    # forced.
+    def _fixture_teardown(self):
+        # Allow TRUNCATE ... CASCADE and don't emit the post_migrate signal
+        # when flushing only a subset of the apps
+        for db_name in self._databases_names(include_mirrors=False):
+            # Flush the database
+            inhibit_post_migrate = (
+                self.available_apps is not None or
+                (   # Inhibit the post_migrate signal when using serialized
+                    # rollback to avoid trying to recreate the serialized data.
+                    self.serialized_rollback and
+                    hasattr(connections[db_name], '_test_serialized_contents')
+                )
+            )
+            call_command('flush', verbosity=0, interactive=False,
+                         database=db_name, reset_sequences=False,
+                         allow_cascade=True,
+                         inhibit_post_migrate=inhibit_post_migrate)
+
+
+def create_remote_webdriver(preset: str = "chrome") -> webdriver.Remote:
+    if preset == "chrome":
+        options = webdriver.ChromeOptions()
+        options.add_argument('--ignore-ssl-errors=yes')
+        options.add_argument('--ignore-certificate-errors')
+        options.add_argument("--window-size=480,720")
+        options.add_argument("--start-maximized")
+        driver = webdriver.Remote(
+            command_executor=get_hub_url(),
+            desired_capabilities=DesiredCapabilities.CHROME,
+            options=options,
+        )
+    elif preset == "firefox":
+        options = webdriver.FirefoxOptions()
+        driver = webdriver.Remote(
+            command_executor=get_hub_url(),
+            desired_capabilities=DesiredCapabilities.FIREFOX,
+            options=options,
+        )
+    else:
+        raise Exception(
+            f"Invalid webdriver preset ('{preset}'); must be 'chrome' or 'firefox'"
+        )
+    driver.set_page_load_timeout(60)
+    driver.implicitly_wait(10)
+    return driver
+
+
+def get_hub_url():
+    return f"http://{settings.SE_HUB_HOST}:{settings.SE_HUB_PORT}/wd/hub"