[Experiment] Switch to Chromium for Integration tests

Dockerfile.integration-tests-debian

@@ -2,8 +2,8 @@
 # code: language=Dockerfile
 
 FROM openapitools/openapi-generator-cli:v7.11.0@sha256:a9e7091ac8808c6835cf8ec88252bca603f1f889ef1456b63d8add5781feeca7 AS openapitools
-# currently only supports x64, no arm yet due to chrome and selenium dependencies
-FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS build
+# ARM64 support inspired by https://github.com/seleniumhq-community/docker-seleniarm/blob/trunk/NodeChromium/Dockerfile to install chromium instead of chrome
+FROM python:3.11.9-slim-bookworm@sha256:6ed5bff4d7d377e2a27d9285553b8c21cfccc4f00881de1b24c9bc8d90016e82 AS base
 WORKDIR /app
 RUN \
   apt-get -y update && \
@@ -18,53 +18,64 @@ RUN \
     apt-file \
     libnss3 \
     xvfb \
+    chromium \
+    chromium-driver \
     && \
-  apt-get clean && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/* \
   true
 
 RUN pip install --no-cache-dir selenium==4.9.0 requests
 
-# Install the latest Google Chrome stable release
-WORKDIR /opt/chrome
 
-# TODO: figure out whatever fix is necessary to use Chrome >= 128 and put this back in the RUN below so we stay
-# up-to-date
-# chrome_url=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels[] | select(.channel == "Stable") | .downloads.chrome[] | select(.platform == "linux64").url') && \
+# root@40b30a85486c:/# which chromium
+# /usr/bin/chromium
 
-RUN \
-  chrome_url="https://storage.googleapis.com/chrome-for-testing-public/127.0.6533.119/linux64/chrome-linux64.zip" && \
-  wget $chrome_url && \
-  unzip chrome-linux64.zip && \
-  rm -rf chrome-linux64.zip && \
-  chmod -R 0755 . && \
-  ln -s /opt/chrome/chrome-linux64/chrome /usr/bin/chrome
-
-# Install the dependencies for Google Chrome
-RUN apt-file update
-COPY docker/install_chrome_dependencies.py install_chrome_dependencies.py
-RUN \
-  missing_chrome_deps=$(python install_chrome_dependencies.py) && \
-  apt-get -y install $missing_chrome_deps
-
-# Install a suggested list of additional packages (https://stackoverflow.com/a/76734752)
-RUN apt-get install -y libxi6 libgconf-2-4 jq libjq1 libonig5 libxkbcommon0 libxss1 libglib2.0-0 libnss3 \
-  libfontconfig1 libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libpango-1.0-0 libgdk-pixbuf2.0-0 libxcomposite1 \
-  libxcursor1 libxdamage1 libxtst6 libappindicator3-1 libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libxfixes3 \
-  libdbus-1-3 libexpat1 libgcc1 libnspr4 libgbm1 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxext6 \
-  libxrandr2 libxrender1 gconf-service ca-certificates fonts-liberation libappindicator1 lsb-release xdg-utils
-
-# Installing the latest stable Google Chrome driver release
-WORKDIR /opt/chrome-driver
-# TODO: figure out whatever fix is necessary to use Chrome >= 128 and put this back in the RUN below so we stay
-# up-to-date
-# chromedriver_url=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels[] | select(.channel == "Stable") | .downloads.chromedriver[] | select(.platform == "linux64").url') && \
+# root@40b30a85486c:/# which chromedriver
+# /usr/bin/chromedriver
 
-RUN \
-  chromedriver_url="https://storage.googleapis.com/chrome-for-testing-public/127.0.6533.119/linux64/chromedriver-linux64.zip" && \
-  wget $chromedriver_url && \
-  unzip -j chromedriver-linux64.zip chromedriver-linux64/chromedriver && \
-  rm -rf chromedriver-linux64.zip && \
-  chmod -R 0755 .
+
+# # Install the latest Google Chrome stable release
+# WORKDIR /opt/chrome
+
+# # TODO: figure out whatever fix is necessary to use Chrome >= 128 and put this back in the RUN below so we stay
+# # up-to-date
+# # chrome_url=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels[] | select(.channel == "Stable") | .downloads.chrome[] | select(.platform == "linux64").url') && \
+
+# RUN \
+#   chrome_url="https://storage.googleapis.com/chrome-for-testing-public/127.0.6533.119/linux64/chrome-linux64.zip" && \
+#   wget $chrome_url && \
+#   unzip chrome-linux64.zip && \
+#   rm -rf chrome-linux64.zip && \
+#   chmod -R 0755 . && \
+#   ln -s /opt/chrome/chrome-linux64/chrome /usr/bin/chrome
+
+# # Install the dependencies for Google Chrome
+# RUN apt-file update
+# COPY docker/install_chrome_dependencies.py install_chrome_dependencies.py
+# RUN \
+#   missing_chrome_deps=$(python install_chrome_dependencies.py) && \
+#   apt-get -y install $missing_chrome_deps
+
+# # Install a suggested list of additional packages (https://stackoverflow.com/a/76734752)
+# RUN apt-get install -y libxi6 libgconf-2-4 jq libjq1 libonig5 libxkbcommon0 libxss1 libglib2.0-0 libnss3 \
+#   libfontconfig1 libatk-bridge2.0-0 libatspi2.0-0 libgtk-3-0 libpango-1.0-0 libgdk-pixbuf2.0-0 libxcomposite1 \
+#   libxcursor1 libxdamage1 libxtst6 libappindicator3-1 libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libxfixes3 \
+#   libdbus-1-3 libexpat1 libgcc1 libnspr4 libgbm1 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxext6 \
+#   libxrandr2 libxrender1 gconf-service ca-certificates fonts-liberation libappindicator1 lsb-release xdg-utils
+
+# # Installing the latest stable Google Chrome driver release
+# WORKDIR /opt/chrome-driver
+# # TODO: figure out whatever fix is necessary to use Chrome >= 128 and put this back in the RUN below so we stay
+# # up-to-date
+# # chromedriver_url=$(curl https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json | jq -r '.channels[] | select(.channel == "Stable") | .downloads.chromedriver[] | select(.platform == "linux64").url') && \
+
+# RUN \
+#   chromedriver_url="https://storage.googleapis.com/chrome-for-testing-public/127.0.6533.119/linux64/chromedriver-linux64.zip" && \
+#   wget $chromedriver_url && \
+#   unzip -j chromedriver-linux64.zip chromedriver-linux64/chromedriver && \
+#   rm -rf chromedriver-linux64.zip && \
+#   chmod -R 0755 .
 WORKDIR /app
 
 # install openapi-generator-cli

docker/entrypoint-integration-tests.sh

@@ -26,9 +26,9 @@ if [ $COUNTER -gt 10 ]; then
     exit 1
 fi
 
-CHROMEDRIVER=$(find /opt/chrome-driver -name chromedriver)
+CHROMEDRIVER=/usr/bin/chromedriver
 export CHROMEDRIVER
-CHROME_PATH=/opt/chrome/chrome
+CHROME_PATH=/usr/bin/chromium
 export CHROME_PATH
 
 # Run available unittests with a simple setup
@@ -251,7 +251,7 @@ else
     else
         fail "$test"
     fi
-    
+
     test="False Positive History tests"
     echo "Running: $test"
     if python3 tests/false_positive_history_test.py ; then

dojo/engagement/views.py

@@ -1586,6 +1586,7 @@ def get_foreign_keys():
 
 
 def csv_export(request):
+    logger.debug("starting csv_export")
     engagements, test_counts = get_engagements(request)
 
     response = HttpResponse(content_type="text/csv")
@@ -1619,10 +1620,12 @@ def csv_export(request):
 
             writer.writerow(fields)
 
+    logger.debug("csv_export done")
     return response
 
 
 def excel_export(request):
+    logger.debug("starting excel_export")
     engagements, test_counts = get_engagements(request)
 
     workbook = Workbook()
@@ -1668,4 +1671,5 @@ def excel_export(request):
         content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
     )
     response["Content-Disposition"] = "attachment; filename=engagements.xlsx"
+    logger.debug("excel_export done")
     return response

tests/Import_scanner_test.py

@@ -13,7 +13,7 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import Select
 
-dir_path = Path(os.path.realpath(__file__)).parent
+dir_path = str(Path(os.path.realpath(__file__)).parent)
 
 logger = logging.getLogger(__name__)
 

tests/close_old_findings_dedupe_test.py

@@ -14,8 +14,6 @@
 
 logger = logging.getLogger(__name__)
 
-dir_path = Path(os.path.realpath(__file__)).parent
-
 
 class CloseOldDedupeTest(BaseTestCase):
     # --------------------------------------------------------------------------------------------------------
@@ -25,7 +23,7 @@ class CloseOldDedupeTest(BaseTestCase):
     # --------------------------------------------------------------------------------------------------------
     def setUp(self):
         super().setUp()
-        self.relative_path = Path(os.path.realpath(__file__)).parent
+        self.relative_path = str(Path(os.path.realpath(__file__)).parent)
 
     def check_nb_duplicates(self, expected_number_of_duplicates):
         logger.debug("checking duplicates...")

tests/close_old_findings_test.py

@@ -13,8 +13,6 @@
 
 logger = logging.getLogger(__name__)
 
-dir_path = Path(os.path.realpath(__file__)).parent
-
 
 class CloseOldTest(BaseTestCase):
     # --------------------------------------------------------------------------------------------------------
@@ -23,7 +21,7 @@ class CloseOldTest(BaseTestCase):
     # --------------------------------------------------------------------------------------------------------
     def setUp(self):
         super().setUp()
-        self.relative_path = Path(os.path.realpath(__file__)).parent
+        self.relative_path = str(Path(os.path.realpath(__file__)).parent)
 
     @on_exception_html_source_logger
     def test_delete_findings(self):

tests/dedupe_test.py

@@ -14,8 +14,6 @@
 
 logger = logging.getLogger(__name__)
 
-dir_path = Path(os.path.realpath(__file__)).parent
-
 
 class DedupeTest(BaseTestCase):
     # --------------------------------------------------------------------------------------------------------

tests/file_test.py

@@ -8,7 +8,7 @@
 from product_test import ProductTest, WaitForPageLoad
 from selenium.webdriver.common.by import By
 
-dir_path = Path(os.path.realpath(__file__)).parent
+dir_path = str(Path(os.path.realpath(__file__)).parent)
 
 
 class FileUploadTest(BaseTestCase):

tests/finding_test.py

@@ -14,7 +14,7 @@
 from user_test import UserTest
 
 logger = logging.getLogger(__name__)
-dir_path = Path(os.path.realpath(__file__)).parent
+dir_path = str(Path(os.path.realpath(__file__)).parent)
 
 
 class FindingTest(BaseTestCase):
@@ -83,7 +83,7 @@ def test_csv_export(self):
         driver.find_element(By.ID, "downloadMenu").click()
         driver.find_element(By.ID, "csv_export").click()
 
-        time.sleep(5)
+        time.sleep(10)
 
         self.check_file(f"{self.export_path}/findings.csv")
 

tests/ibm_appscan_test.py

@@ -8,7 +8,7 @@
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support.ui import Select
 
-dir_path = Path(os.path.realpath(__file__)).parent
+dir_path = str(Path(os.path.realpath(__file__)).parent)
 
 
 class IBMAppScanTest(BaseTestCase):

tests/report_builder_test.py

@@ -1,7 +1,5 @@
-import os
 import sys
 import unittest
-from pathlib import Path
 
 from base_test_class import BaseTestCase
 from product_test import ProductTest
@@ -10,8 +8,6 @@
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import Select, WebDriverWait
 
-dir_path = Path(os.path.realpath(__file__)).parent
-
 
 class ReportBuilderTest(BaseTestCase):