Merge branch 'DefectDojo:dev' into dev

Author: jawadqur

.github/ISSUE_TEMPLATE/bug_report.md

@@ -36,7 +36,7 @@ A clear and concise description of what you expected to happen.
  - DefectDojo version (see footer) or commit message: [use `git show -s --format="[%ci] %h: %s [%d]"`]
 
 **Logs** 
-Use `docker-compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
+Use `docker compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
 
 **Sample scan files**
 If applicable, add sample scan files to help reproduce your problem.

.github/ISSUE_TEMPLATE/support_request.md

@@ -36,7 +36,7 @@ A clear and concise description of what you expected to happen.
  - DefectDojo version (see footer) or commit message: [use `git show -s --format="[%ci] %h: %s [%d]"`]
 
 **Logs** 
-Use `docker-compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
+Use `docker compose logs` (or similar, depending on your deployment method) to get the logs and add the relevant sections here showing the error occurring (if applicable).
 
 **Sample scan files**
 If applicable, add sample scan files to help reproduce your problem.

.github/workflows/build-docker-images-for-testing.yml

@@ -45,9 +45,7 @@ jobs:
           tags: defectdojo/defectdojo-${{ matrix.docker-image }}:${{ matrix.os }}
           file: Dockerfile.${{ matrix.docker-image }}-${{ matrix.os }}
           outputs: type=docker,dest=${{ matrix.docker-image }}-${{ matrix.os }}_img
-          cache-from: type=gha,scope=${{ matrix.docker-image }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.docker-image }}
-  
+
       # export docker images to be used in next jobs below
       - name: Upload image ${{ matrix.docker-image }} as artifact
         timeout-minutes: 10

.github/workflows/k8s-tests.yml

@@ -35,7 +35,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Minikube
-        uses: manusa/actions-setup-minikube@v2.11.0
+        uses: manusa/actions-setup-minikube@v2.13.0
         with:
           minikube version: 'v1.33.1'
           kubernetes version: ${{ matrix.k8s }}

.github/workflows/release-3-master-into-dev.yml

@@ -50,11 +50,15 @@ jobs:
           CURRENT_CHART_VERSION=$(grep -oP 'version: (\K\S*)?' helm/defectdojo/Chart.yaml | head -1)
           sed -ri "0,/version/s/version: \S+/$(echo "version: $CURRENT_CHART_VERSION" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')-dev/" helm/defectdojo/Chart.yaml
       
+      - name: Update settings SHA
+        run: sha256sum dojo/settings/settings.dist.py | cut -d ' ' -f1 > dojo/settings/.settings.dist.py.sha256sum
+
       - name: Check numbers
         run: |
           grep version dojo/__init__.py
           grep appVersion helm/defectdojo/Chart.yaml
           grep version components/package.json
+          cat dojo/settings/.settings.dist.py.sha256sum
 
       - name: Create upgrade notes to documentation
         run: |
@@ -132,11 +136,15 @@ jobs:
           CURRENT_CHART_VERSION=$(grep -oP 'version: (\K\S*)?' helm/defectdojo/Chart.yaml | head -1)
           sed -ri "0,/version/s/version: \S+/$(echo "version: $CURRENT_CHART_VERSION" | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')-dev/" helm/defectdojo/Chart.yaml
       
+      - name: Update settings SHA
+        run: sha256sum dojo/settings/settings.dist.py | cut -d ' ' -f1 > dojo/settings/.settings.dist.py.sha256sum
+
       - name: Check numbers
         run: |
           grep version dojo/__init__.py
           grep appVersion helm/defectdojo/Chart.yaml
           grep version components/package.json
+          cat dojo/settings/.settings.dist.py.sha256sum
       
       - name: Push version changes
         uses: stefanzweifel/[email protected]

.github/workflows/release-x-manual-docker-containers.yml

@@ -49,18 +49,6 @@ jobs:
         id: buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Cache Docker layers
-        uses: actions/cache@v4
-        env:
-          docker-image: ${{ matrix.docker-image }}
-        with:
-          path: /tmp/.buildx-cache-${{ env.docker-image }}
-          key: ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ matrix.os }}-${{ env.workflow_name }}-${{ github.sha }}-${{ github.run_id }}
-          restore-keys: |
-            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ matrix.os }}-${{ env.workflow_name}}-${{ github.sha }}
-            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ matrix.os }}-${{ env.workflow_name }}
-            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ matrix.os }}-
-
       - name: Build and push images with debian
         if: ${{ matrix.os  == 'debian' }}
         uses: docker/build-push-action@v6
@@ -73,8 +61,6 @@ jobs:
           tags: ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:${{ github.event.inputs.release_number }}-${{ matrix.os }}, ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:${{ github.event.inputs.release_number }}, ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:latest
           file: ./Dockerfile.${{ env.docker-image }}-${{ matrix.os }}
           context: .
-          cache-from: type=local,src=/tmp/.buildx-cache-${{ env.docker-image }}
-          cache-to: type=local,dest=/tmp/.buildx-cache-${{ env.docker-image }}
 
       - name: Build and push images with alpine
         if: ${{ matrix.os  == 'alpine' }}
@@ -88,9 +74,3 @@ jobs:
           tags: ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:${{ github.event.inputs.release_number }}-${{ matrix.os }}
           file: ./Dockerfile.${{ env.docker-image }}-${{ matrix.os }}
           context: .
-          cache-from: type=local,src=/tmp/.buildx-cache-${{ env.docker-image }}
-          cache-to: type=local,dest=/tmp/.buildx-cache-${{ env.docker-image }}
-#           platforms: ${{ matrix.platform }}
-
-      - name: Image digest
-        run: echo ${{ steps.docker_build.outputs.digest }}

.github/workflows/rest-framework-tests.yml

@@ -34,8 +34,8 @@ jobs:
         run: docker/setEnv.sh unit_tests_cicd
 
       # phased startup so we can use the exit code from unit test container
-      - name: Start Postgres
-        run: docker compose up -d postgres
+      - name: Start Postgres and webhook.endpoint
+        run: docker compose up -d postgres webhook.endpoint
 
       # no celery or initializer needed for unit tests
       - name: Unit tests

Dockerfile.integration-tests-debian

@@ -1,7 +1,7 @@
 
 # code: language=Dockerfile
 
-FROM openapitools/openapi-generator-cli:v7.8.0@sha256:c409bfa9b276faf27726d2884b859d18269bf980cb63546e80b72f3b2648c492 AS openapitools
+FROM openapitools/openapi-generator-cli:v7.9.0@sha256:bb32f5f0c9f5bdbb7b00959e8009de0230aedc200662701f05fc244c36f967ba AS openapitools
 FROM python:3.11.9-slim-bookworm@sha256:8c1036ec919826052306dfb5286e4753ffd9d5f6c24fbc352a5399c3b405b57e AS build
 WORKDIR /app
 RUN \

Dockerfile.nginx-alpine

@@ -140,7 +140,7 @@ COPY manage.py ./
 COPY dojo/ ./dojo/
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.27.0-alpine@sha256:208b70eefac13ee9be00e486f79c695b15cef861c680527171a27d253d834be9
+FROM nginx:1.27.2-alpine@sha256:2140dad235c130ac861018a4e13a6bc8aea3a35f3a40e20c1b060d51a7efd250
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

Dockerfile.nginx-debian

@@ -73,7 +73,7 @@ COPY dojo/ ./dojo/
 
 RUN env DD_SECRET_KEY='.' python3 manage.py collectstatic --noinput && true
 
-FROM nginx:1.27.0-alpine@sha256:208b70eefac13ee9be00e486f79c695b15cef861c680527171a27d253d834be9
+FROM nginx:1.27.2-alpine@sha256:2140dad235c130ac861018a4e13a6bc8aea3a35f3a40e20c1b060d51a7efd250
 ARG uid=1001
 ARG appuser=defectdojo
 COPY --from=collectstatic /app/static/ /usr/share/nginx/html/static/

README.md

@@ -132,15 +132,14 @@ Core Moderators can help you with pull requests or feedback on dev ideas:
 * Cody Maffucci ([@Maffooch](https://github.com/maffooch) | [LinkedIn](https://www.linkedin.com/in/cody-maffucci))
 
 Moderators can help you with pull requests or feedback on dev ideas:
-* Damien Carol ([@damiencarol](https://github.com/damiencarol) | [LinkedIn](https://www.linkedin.com/in/damien-carol/))
-* Jannik Jürgens ([@alles-klar](https://github.com/alles-klar))
-* Dubravko Sever ([@dsever](https://github.com/dsever))
 * Charles Neill ([@cneill](https://github.com/cneill) | [@ccneill](https://twitter.com/ccneill))
 * Jay Paz ([@jjpaz](https://twitter.com/jjpaz))
 * Blake Owens ([@blakeaowens](https://github.com/blakeaowens))
 
 ## Hall of Fame
-
+* Jannik Jürgens ([@alles-klar](https://github.com/alles-klar)) - Jannik was a long time contributor and moderator for 
+  DefectDojo and made significant contributions to many areas of the platform. Jannik was instrumental in pioneering 
+  and optimizing deployment methods.
 * Valentijn Scholten ([@valentijnscholten](https://github.com/valentijnscholten) |
   [Sponsor](https://github.com/sponsors/valentijnscholten) |
   [LinkedIn](https://www.linkedin.com/in/valentijn-scholten/)) - Valentijn served as a core moderator for 3 years.

components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "defectdojo",
-  "version": "2.39.0-dev",
+  "version": "2.40.0-dev",
   "license" : "BSD-3-Clause",
   "private": true,
   "dependencies": {
@@ -35,7 +35,7 @@
     "metismenu": "~3.0.7",
     "moment": "^2.30.1",
     "morris.js": "morrisjs/morris.js",
-    "pdfmake": "^0.2.12",
+    "pdfmake": "^0.2.14",
     "startbootstrap-sb-admin-2": "1.0.7"
   },
   "engines": {

components/yarn.lock

@@ -824,10 +824,10 @@ path-parse@^1.0.7:
   resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.7.tgz#fbc114b60ca42b30d9daf5858e4bd68bbedb6735"
   integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==
 
-pdfmake@^0.2.12:
-  version "0.2.12"
-  resolved "https://registry.yarnpkg.com/pdfmake/-/pdfmake-0.2.12.tgz#5156f91ff73797947942aa342423bedaa0c0bc93"
-  integrity sha512-TFsqaG6KVtk+TWermmJNNwom3wmB/xiz07prM74KBhdM+7pz3Uwq2b0uoqhhQRn6cYUTpL8lXZY6xF011o1YcQ==
+pdfmake@^0.2.14:
+  version "0.2.14"
+  resolved "https://registry.yarnpkg.com/pdfmake/-/pdfmake-0.2.14.tgz#a257a393b54917218add829bff8e490be21e8077"
+  integrity sha512-x9gXFAY37/CAC/WaZB/683E4Pi0cVW/RMTTNxMpe4I2kRsKv8AE3Pz6+n7iTfn+84/GtSg99BjZkYh7oGFCKmg==
   dependencies:
     "@foliojs-fork/linebreak" "^1.1.1"
     "@foliojs-fork/pdfkit" "^0.14.0"

docker-compose.override.dev.yml

@@ -53,3 +53,5 @@ services:
         published: 8025
         protocol: tcp
         mode: host
+  "webhook.endpoint":
+    image: mccutchen/go-httpbin:v2.15.0@sha256:24528cf5229d0b70065ac27e6c9e4d96f5452a84a3ce4433e56573c18d96827a

docker-compose.override.unit_tests.yml

@@ -1,7 +1,7 @@
 ---
 services:
   nginx:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'nginx']
     volumes:
       - defectdojo_media_unit_tests:/usr/share/nginx/html/media
@@ -30,13 +30,13 @@ services:
       DD_CELERY_BROKER_PATH: '/dojo.celerydb.sqlite'
       DD_CELERY_BROKER_PARAMS: ''
   celerybeat:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'celery beat']
   celeryworker:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'celery worker']
   initializer:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'initializer']
   postgres:
     ports:
@@ -49,8 +49,10 @@ services:
     volumes:
       - defectdojo_postgres_unit_tests:/var/lib/postgresql/data
   redis:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'redis']
+  "webhook.endpoint":
+    image: mccutchen/go-httpbin:v2.15.0@sha256:24528cf5229d0b70065ac27e6c9e4d96f5452a84a3ce4433e56573c18d96827a
 volumes:
   defectdojo_postgres_unit_tests: {}
   defectdojo_media_unit_tests: {}

docker-compose.override.unit_tests_cicd.yml

@@ -1,7 +1,7 @@
 ---
 services:
   nginx:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'nginx']
     volumes:
       - defectdojo_media_unit_tests:/usr/share/nginx/html/media
@@ -29,13 +29,13 @@ services:
       DD_CELERY_BROKER_PATH: '/dojo.celerydb.sqlite'
       DD_CELERY_BROKER_PARAMS: ''
   celerybeat:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'celery beat']
   celeryworker:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'celery worker']
   initializer:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'initializer']
   postgres:
     ports:
@@ -48,8 +48,10 @@ services:
     volumes:
       - defectdojo_postgres_unit_tests:/var/lib/postgresql/data
   redis:
-    image: busybox:1.36.1-musl
+    image: busybox:1.37.0-musl
     entrypoint: ['echo', 'skipping', 'redis']
+  "webhook.endpoint":
+    image: mccutchen/go-httpbin:v2.15.0@sha256:24528cf5229d0b70065ac27e6c9e4d96f5452a84a3ce4433e56573c18d96827a
 volumes:
   defectdojo_postgres_unit_tests: {}
   defectdojo_media_unit_tests: {}

docker-compose.yml

@@ -103,15 +103,15 @@ services:
           source: ./docker/extra_settings
           target: /app/docker/extra_settings
   postgres:
-    image: postgres:16.4-alpine@sha256:492898505cb45f9835acc327e98711eaa9298ed804e0bb36f29e08394229550d
+    image: postgres:17.0-alpine@sha256:14195b0729fce792f47ae3c3704d6fd04305826d57af3b01d5b4d004667df174
     environment:
       POSTGRES_DB: ${DD_DATABASE_NAME:-defectdojo}
       POSTGRES_USER: ${DD_DATABASE_USER:-defectdojo}
       POSTGRES_PASSWORD: ${DD_DATABASE_PASSWORD:-defectdojo}
     volumes:
       - defectdojo_postgres:/var/lib/postgresql/data
   redis:
-    image: redis:7.2.5-alpine@sha256:0bc09d9f486508aa42ecc2f18012bb1e3a1b2744ef3a6ad30942fa12579f0b03
+    image: redis:7.2.5-alpine@sha256:6aaf3f5e6bc8a592fbfe2cccf19eb36d27c39d12dab4f4b01556b7449e7b1f44
     volumes:
       - defectdojo_redis:/data
 volumes:

docker/docker-compose-check.sh

@@ -6,11 +6,11 @@ current=$(docker compose  version  --short)
 
 echo 'Checking docker compose version'
 if [[ $main -lt 2 ]]; then
-  echo "$current is not a supported docker-compose version, please upgrade to the minimum supported version: 2.0"
+  echo "$current is not a supported 'docker compose' version, please upgrade to the minimum supported version: 2.0"
   exit 1
 elif [[ $main -eq 1 ]]; then
   if [[ $minor -lt 28 ]]; then
-    echo "$current is not supported docker-compose version, please upgrade to minimal supported version:1.28"
+    echo "$current is not supported 'docker compose' version, please upgrade to minimal supported version:1.28"
     exit 1
   fi
 fi

docker/extra_settings/README.md

@@ -6,7 +6,7 @@ If a file if placed here, it will be copied on startup to `dojo/settings/local_s
 For an example, see [template-local_settings](../../dojo/settings/template-local_settings)
 
 Please note this copy action could fail if you have mounted the full `dojo/` folder, but that is owned by a different user/group.
-That's why this copy action only happens in docker-compose release mode, and not in dev/debug/unit_tests/integration_tests modes.
+That's why this copy action only happens in docker compose release mode, and not in dev/debug/unit_tests/integration_tests modes.
 
 For advanced usage you can also place a `settings.dist.py` or `settings.py` file. These will also be copied on startup to dojo/settings.
 

docker/install_chrome_dependencies.py

@@ -10,43 +10,47 @@
 
 
 def find_packages(library_name):
-    stdout = run_command(["apt-file", "search", library_name])
+    stdout, stderr, status_code = run_command(["apt-file", "search", library_name])
+    # Check if ldd has failed for a good reason, or if there are no results
+    if status_code != 0:
+        # Any other case should be be caught
+        msg = f"apt-file search (exit code {status_code}): {stderr}"
+        raise ValueError(msg)
+
     if not stdout.strip():
         return []
     libs = [line.split(":")[0] for line in stdout.strip().split("\n")]
     return list(set(libs))
 
 
 def run_command(cmd, cwd=None, env=None):
+    # Do not raise exception here because some commands are too loose with negative exit codes
     result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=False)
-    return result.stdout
+    return result.stdout.strip(), result.stderr.strip(), result.returncode
 
 
 def ldd(file_path):
-    stdout = run_command(["ldd", file_path])
-    # For simplicity, I'm assuming if we get an error, the code is non-zero.
-    try:
-        result = subprocess.run(
-            ["ldd", file_path], capture_output=True, text=True, check=False,
-        )
-        stdout = result.stdout
-        code = result.returncode
-    except subprocess.CalledProcessError:
-        stdout = ""
-        code = 1
-    return stdout, code
+    stdout, stderr, status_code = run_command(["ldd", file_path])
+    # Check if ldd has failed for a good reason, or if there are no results
+    if status_code != 0:
+        # It is often the case when stdout will be empty. This is not an error
+        if not stdout:
+            return stdout, status_code
+        # Any other case should be be caught
+        msg = f"ldd (exit code {status_code}): {stderr}"
+        raise ValueError(msg)
+
+    return stdout, status_code
 
 
 raw_deps = ldd("/opt/chrome/chrome")
 dependencies = raw_deps[0].splitlines()
-
 missing_deps = {
     r[0].strip()
     for d in dependencies
     for r in [d.split("=>")]
     if len(r) == 2 and r[1].strip() == "not found"
 }
-
 missing_packages = []
 for d in missing_deps:
     all_packages = find_packages(d)
@@ -59,5 +63,4 @@ def ldd(file_path):
     ]
     for p in packages:
         missing_packages.append(p)
-
 logger.info("missing_packages: " + (" ".join(missing_packages)))