Merge pull request #12755 from DefectDojo/bugfix

 Release 2.48.0: Merge Bugfix into Dev

Author: rossops

.github/workflows/integration-tests.yml

@@ -31,10 +31,17 @@ jobs:
           "tests/search_test.py",
           "tests/file_test.py",
           "tests/dedupe_test.py",
+          "tests/announcement_banner_test.py",
+          "tests/close_old_findings_dedupe_test.py",
+          "tests/close_old_findings_test.py",
+          "tests/false_positive_history_test.py",
           "tests/check_various_pages.py",
+          # "tests/import_scanner_test.py",
+          # "tests/zap.py",
           "tests/notifications_test.py",
           "tests/tool_config.py",
           "openapi-validatator",
+
         ]
         os: [alpine, debian]
       fail-fast: false

Dockerfile.django-alpine

@@ -67,6 +67,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
+  docker/entrypoint-celery-worker-dev.sh \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-first-boot.sh \
   docker/entrypoint-uwsgi.sh \

Dockerfile.django-debian

@@ -70,6 +70,7 @@ RUN export PYCURL_SSL_LIBRARY=openssl && \
 COPY \
   docker/entrypoint-celery-beat.sh \
   docker/entrypoint-celery-worker.sh \
+  docker/entrypoint-celery-worker-dev.sh \
   docker/entrypoint-initializer.sh \
   docker/entrypoint-first-boot.sh \
   docker/entrypoint-uwsgi.sh \

docker-compose.override.dev.yml

@@ -11,6 +11,7 @@ services:
       DD_ADMIN_PASSWORD: "${DD_ADMIN_PASSWORD:-admin}"
       DD_EMAIL_URL: "smtp://mailhog:1025"
   celeryworker:
+    entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh']
     volumes:
       - '.:/app:z'
     environment:

docker-compose.override.integration_tests.yml

@@ -38,6 +38,7 @@ services:
     environment:
       DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo}
   celeryworker:
+    entrypoint: ['/wait-for-it.sh', '${DD_DATABASE_HOST:-postgres}:${DD_DATABASE_PORT:-5432}', '-t', '30', '--', '/entrypoint-celery-worker-dev.sh']
     environment:
       DD_DATABASE_URL: ${DD_TEST_DATABASE_URL:-postgresql://defectdojo:defectdojo@postgres:5432/test_defectdojo}
   initializer:

docker/entrypoint-celery-worker-dev.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+umask 0002
+
+id
+
+set -e  # needed to handle "exit" correctly
+
+. /secret-file-loader.sh
+. /reach_database.sh
+
+wait_for_database_to_be_reachable
+echo
+
+if [ "${DD_CELERY_WORKER_POOL_TYPE}" = "prefork" ]; then
+  EXTRA_PARAMS=("--autoscale=${DD_CELERY_WORKER_AUTOSCALE_MAX},${DD_CELERY_WORKER_AUTOSCALE_MIN}"
+    "--prefetch-multiplier=${DD_CELERY_WORKER_PREFETCH_MULTIPLIER}")
+else
+  EXTRA_PARAMS=()
+fi
+
+# do the check with Django stack
+python3 manage.py check
+
+# hot reload using watmedo as we don't want to install celery[dev] and have that end up in our production images
+watchmedo auto-restart --directory=./ --pattern="*.py;*.tpl" --recursive -- \
+  celery --app=dojo worker --loglevel="${DD_CELERY_LOG_LEVEL}" --pool="${DD_CELERY_WORKER_POOL_TYPE}" --concurrency="${DD_CELERY_WORKER_CONCURRENCY:-1}" "${EXTRA_PARAMS[@]}"

docker/entrypoint-integration-tests.sh

@@ -288,6 +288,14 @@ else
     #     echo "Error: Zap integration test failed"; exit 1
     # fi
 
+    test="Notifications tests"
+    echo "Running: $test"
+    if python3 tests/notifications_test.py ; then
+        success "$test"
+    else
+        fail "$test"
+    fi
+
     test="Tool Config integration tests"
     echo "Running: $test"
     if python3 tests/tool_config.py ; then

docker/entrypoint-uwsgi-dev.sh

@@ -21,6 +21,14 @@ if [ "${DD_DEBUG}" = "True" ]; then
   DD_UWSGI_NUM_OF_THREADS=1
 fi
 
+# hot reload also on html/template changes
+watchmedo shell-command \
+  --patterns="*.html;*.tpl" \
+  --recursive \
+  --command='touch /app/dojo/settings/settings.py' \
+  /app/dojo &
+
+
 exec uwsgi \
   "--${DD_UWSGI_MODE}" "${DD_UWSGI_ENDPOINT}" \
   --protocol uwsgi \
@@ -33,5 +41,5 @@ exec uwsgi \
   --py-autoreload 1 \
   --buffer-size="${DD_UWSGI_BUFFER_SIZE:-8192}" \
   --lazy-apps \
-  --touch-reload="/app/dojo/setting/settings.py" \
+  --touch-reload="/app/dojo/settings/settings.py" \
   --logformat "${DD_UWSGI_LOGFORMAT:-$DD_UWSGI_LOGFORMAT_DEFAULT}"

dojo/decorators.py

@@ -155,7 +155,7 @@ def __wrapper__(*args, **kwargs):
                     try:
                         instance = model.objects.get(id=model_or_id)
                     except model.DoesNotExist:
-                        logger.debug("error instantiating model_or_id: %s for model: %s: DoesNotExist", model_or_id, model)
+                        logger.warning("error instantiating model_or_id: %s for model: %s: DoesNotExist", model_or_id, model)
                         instance = None
                     args = list(args)
                     args[parameter] = instance

dojo/filters.py

@@ -1975,6 +1975,7 @@ def __init__(self, *args, **kwargs):
         self.set_related_object_fields(*args, **kwargs)
 
     def set_related_object_fields(self, *args: list, **kwargs: dict):
+        finding_group_query = Finding_Group.objects.all()
         if self.pid is not None:
             del self.form.fields["test__engagement__product"]
             del self.form.fields["test__engagement__product__prod_type"]
@@ -1983,6 +1984,7 @@ def set_related_object_fields(self, *args: list, **kwargs: dict):
                 product_id=self.pid,
             ).all()
             self.form.fields["test"].queryset = get_authorized_tests(Permissions.Test_View, product=self.pid).prefetch_related("test_type")
+            finding_group_query = Finding_Group.objects.filter(test__engagement__product_id=self.pid)
         else:
             self.form.fields[
                 "test__engagement__product__prod_type"].queryset = get_authorized_product_types(Permissions.Product_Type_View)
@@ -1992,7 +1994,7 @@ def set_related_object_fields(self, *args: list, **kwargs: dict):
         if self.form.fields.get("test__engagement__product"):
             self.form.fields["test__engagement__product"].queryset = get_authorized_products(Permissions.Product_View)
         if self.form.fields.get("finding_group", None):
-            self.form.fields["finding_group"].queryset = get_authorized_finding_groups(Permissions.Finding_Group_View)
+            self.form.fields["finding_group"].queryset = get_authorized_finding_groups(Permissions.Finding_Group_View, queryset=finding_group_query)
         self.form.fields["reporter"].queryset = get_authorized_users(Permissions.Finding_View)
         self.form.fields["reviewers"].queryset = self.form.fields["reporter"].queryset