Merge pull request #2169 from aboutcode-org/advisory_grouping

Group related advisories on basis of content

Author: TG1999

vulnerabilities/api_v2.py

@@ -41,6 +41,7 @@
 from vulnerabilities.models import VulnerabilitySeverity
 from vulnerabilities.models import Weakness
 from vulnerabilities.throttling import PermissionBasedUserRateThrottle
+from vulnerabilities.utils import group_advisories_by_content
 
 
 class CharInFilter(filters.BaseInFilter, filters.CharFilter):
@@ -361,19 +362,39 @@ def get_affected_by_vulnerabilities(self, package):
 
         latest_advisories = AdvisoryV2.objects.latest_for_avids(avids)
         advisory_by_avid = {adv.avid: adv for adv in latest_advisories}
+        impact_by_avid = {}
 
-        result = {}
-
+        advisories = []
         for impact in impacts:
             avid = impact.advisory.avid
             advisory = advisory_by_avid.get(avid)
             if not advisory:
                 continue
-            fixed_by_packages = [pkg.purl for pkg in impact.fixed_by_packages.all()]
-            result[advisory.avid] = {
-                "advisory_id": advisory.avid,
-                "fixed_by_packages": fixed_by_packages,
-            }
+            advisories.append(advisory)
+            impact_by_avid[avid] = impact
+
+        grouped_advisories = group_advisories_by_content(advisories=advisories)
+
+        advs = []
+
+        for hash in grouped_advisories:
+            advs.append(grouped_advisories[hash])
+
+        result = []
+
+        for advisory in advs:
+            primary_advisory = advisory["primary"]
+            avid = primary_advisory.avid
+            impact = impact_by_avid.get(avid)
+            if not impact:
+                continue
+            result.append(
+                {
+                    "advisory_id": primary_advisory.avid,
+                    "fixed_by_packages": [pkg.purl for pkg in impact.fixed_by_packages.all()],
+                    "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]],
+                }
+            )
 
         return result
 
@@ -384,7 +405,25 @@ def get_fixing_vulnerabilities(self, package):
 
         latest_advisories = AdvisoryV2.objects.latest_for_avids(avids)
 
-        return [adv.avid for adv in latest_advisories]
+        grouped_advisories = group_advisories_by_content(advisories=latest_advisories)
+
+        advs = []
+
+        for hash in grouped_advisories:
+            advs.append(grouped_advisories[hash])
+
+        result = []
+
+        for advisory in advs:
+            primary_advisory = advisory["primary"]
+            result.append(
+                {
+                    "advisory_id": primary_advisory.avid,
+                    "duplicate_advisory_ids": [adv.avid for adv in advisory["secondary"]],
+                }
+            )
+
+        return result
 
     def get_next_non_vulnerable_version(self, package):
         if next_non_vulnerable := package.get_non_vulnerable_versions()[0]:
@@ -1078,14 +1117,14 @@ def list(self, request, *args, **kwargs):
             return self.get_paginated_response(
                 {
                     "packages": serializer.data,
-                    "advisories": advisory_data,
+                    "advisories_by_id": advisory_data,
                 }
             )
 
         return Response(
             {
                 "packages": serializer.data,
-                "advisories": advisory_data,
+                "advisories_by_id": advisory_data,
             }
         )
 
@@ -1160,7 +1199,7 @@ def bulk_lookup(self, request):
         return Response(
             {
                 "packages": package_data,
-                "advisories": advisory_data,
+                "advisories_by_id": advisory_data,
             }
         )
 
@@ -1254,7 +1293,7 @@ def bulk_search(self, request):
                 return Response(
                     {
                         "packages": package_data,
-                        "advisories": advisory_data,
+                        "advisories_by_id": advisory_data,
                     }
                 )
 
@@ -1308,7 +1347,7 @@ def bulk_search(self, request):
             return Response(
                 {
                     "packages": package_data,
-                    "advisories": advisory_data,
+                    "advisories_by_id": advisory_data,
                 }
             )
 

vulnerabilities/migrations/0113_advisoryv2_precedence.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.25 on 2026-02-10 12:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0112_alter_advisoryseverity_scoring_system_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="precedence",
+            field=models.IntegerField(
+                blank=True,
+                help_text="Precedence indicates the priority of advisory from different datasources. It is determined based on the reliability of the datasource and how close it is to the source.",
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -10,6 +10,7 @@
 import csv
 import datetime
 import hashlib
+import json
 import logging
 import uuid
 import xml.etree.ElementTree as ET
@@ -69,7 +70,9 @@
 from vulnerabilities.severity_systems import EPSS
 from vulnerabilities.severity_systems import SCORING_SYSTEMS
 from vulnerabilities.utils import compute_patch_checksum
+from vulnerabilities.utils import normalize_list
 from vulnerabilities.utils import normalize_purl
+from vulnerabilities.utils import normalize_text
 from vulnerabilities.utils import purl_to_dict
 from vulnerablecode import __version__ as VULNERABLECODE_VERSION
 from vulnerablecode.settings import VULNERABLECODE_PIPELINE_TIMEOUT
@@ -2988,11 +2991,11 @@ class AdvisoryV2(models.Model):
         help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.",
     )
 
-    # precedence = models.IntegerField(
-    #     null=True,
-    #     blank=True,
-    #     help_text="Precedence indicates the priority level of addressing a vulnerability based on its overall risk",
-    # )
+    precedence = models.IntegerField(
+        null=True,
+        blank=True,
+        help_text="Precedence indicates the priority of advisory from different datasources. It is determined based on the reliability of the datasource and how close it is to the source.",
+    )
 
     @property
     def risk_score(self):
@@ -3038,16 +3041,20 @@ def to_advisory_data(self) -> "AdvisoryDataV2":
 
         return AdvisoryDataV2(
             advisory_id=self.advisory_id,
-            aliases=[item.alias for item in self.aliases.all()],
+            aliases=normalize_list([item.alias for item in self.aliases.all()]),
             summary=self.summary,
-            affected_packages=[
-                impacted.to_affected_package_data() for impacted in self.impacted_packages.all()
-            ],
-            references=[ref.to_reference_v2_data() for ref in self.references.all()],
-            patches=[patch.to_patch_data() for patch in self.patches.all()],
+            affected_packages=normalize_list(
+                [impacted.to_affected_package_data() for impacted in self.impacted_packages.all()]
+            ),
+            references=normalize_list(
+                [ref.to_reference_v2_data() for ref in self.references.all()]
+            ),
+            patches=normalize_list([patch.to_patch_data() for patch in self.patches.all()]),
             date_published=self.date_published,
-            weaknesses=[weak.cwe_id for weak in self.weaknesses.all()],
-            severities=[sev.to_vulnerability_severity_data() for sev in self.severities.all()],
+            weaknesses=normalize_list([weak.cwe_id for weak in self.weaknesses.all()]),
+            severities=normalize_list(
+                [sev.to_vulnerability_severity_data() for sev in self.severities.all()]
+            ),
             url=self.url,
         )
 
@@ -3058,6 +3065,35 @@ def get_aliases(self):
         """
         return self.aliases.all()
 
+    def compute_advisory_content(self):
+        """
+        Compute a unique content hash for an advisory by normalizing its data and hashing it.
+
+        :param advisory: An Advisory object
+        :return: SHA-256 hash digest as content hash
+        """
+        normalized_data = {
+            "summary": normalize_text(self.summary),
+            "impacted_packages": sorted(
+                [impact.to_dict() for impact in self.impacted_packages.all()],
+                key=lambda x: json.dumps(x, sort_keys=True),
+            ),
+            "patches": sorted(
+                [patch.to_patch_data().to_dict() for patch in self.patches.all()],
+                key=lambda x: json.dumps(x, sort_keys=True),
+            ),
+            "severities": sorted(
+                [sev.to_vulnerability_severity_data().to_dict() for sev in self.severities.all()],
+                key=lambda x: (x.get("system"), x.get("value")),
+            ),
+            "weaknesses": normalize_list([weakness.cwe_id for weakness in self.weaknesses.all()]),
+        }
+
+        normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True)
+        content_hash = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest()
+
+        return content_hash
+
     alias = get_aliases
 
 

vulnerabilities/pipelines/__init__.py

@@ -265,6 +265,7 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
     spdx_license_expression = None
     repo_url = None
     ignorable_versions = []
+    precedence = 0
 
     # Control how often progress log is shown (range: 1–100, higher value = less frequent log)
     progress_step = 10
@@ -318,6 +319,7 @@ def collect_and_store_advisories(self):
                     advisory=advisory,
                     pipeline_id=self.pipeline_id,
                     logger=self.log,
+                    precedence=self.precedence,
                 ):
                     collected_advisory_count += 1
             except Exception as e:

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -38,6 +38,8 @@ class AlpineLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://secdb.alpinelinux.org/license.txt"
     repo_url = "git+https://github.com/aboutcode-org/aboutcode-mirror-alpine-secdb/"
 
+    precedence = 200
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -31,6 +31,8 @@ class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "Apache-2.0"
     license_url = "https://github.com/quarkslab/aosp_dataset/blob/master/LICENSE"
 
+    precedence = 200
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_importers/apache_httpd_importer.py

@@ -152,6 +152,8 @@ class ApacheHTTPDImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://httpd.apache.org/security/json/"
 
+    precedence = 200
+
     links = []
 
     ignorable_versions = frozenset(

vulnerabilities/pipelines/v2_importers/apache_kafka_importer.py

@@ -48,6 +48,8 @@ class ApacheKafkaImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
         "CVE-2021-4104",
     ]
 
+    precedence = 200
+
     @classmethod
     def steps(cls):
         return (

vulnerabilities/pipelines/v2_importers/apache_tomcat_importer.py

@@ -40,6 +40,8 @@ class ApacheTomcatImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     license_url = "https://www.apache.org/licenses/LICENSE-2.0"
     base_url = "https://tomcat.apache.org/security"
 
+    precedence = 200
+
     def fetch_advisory_links(self):
         """
         Yield the URLs of each Tomcat version security-related page.

vulnerabilities/pipelines/v2_importers/archlinux_importer.py

@@ -28,6 +28,8 @@ class ArchLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
     spdx_license_expression = "MIT"
     license_url = "https://github.com/archlinux/arch-security-tracker/blob/master/LICENSE"
 
+    precedence = 200
+
     @classmethod
     def steps(cls):
         return (