Migrate Gentoo importer to advisory V2

vulnerabilities/importers/__init__.py

@@ -49,6 +49,7 @@
     elixir_security_importer as elixir_security_importer_v2,
 )
 from vulnerabilities.pipelines.v2_importers import epss_importer_v2
+from vulnerabilities.pipelines.v2_importers import gentoo_importer as gentoo_importer_v2
 from vulnerabilities.pipelines.v2_importers import github_osv_importer as github_osv_importer_v2
 from vulnerabilities.pipelines.v2_importers import gitlab_importer as gitlab_importer_v2
 from vulnerabilities.pipelines.v2_importers import istio_importer as istio_importer_v2
@@ -88,6 +89,7 @@
         aosp_importer_v2.AospImporterPipeline,
         ruby_importer_v2.RubyImporterPipeline,
         epss_importer_v2.EPSSImporterPipeline,
+        gentoo_importer_v2.GentooImporterPipeline,
         mattermost_importer_v2.MattermostImporterPipeline,
         nvd_importer.NVDImporterPipeline,
         github_importer.GitHubAPIImporterPipeline,

vulnerabilities/pipelines/v2_importers/gentoo_importer.py

@@ -0,0 +1,192 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Iterable
+
+from fetchcode.vcs import fetch_via_vcs
+from packageurl import PackageURL
+from univers.version_constraint import VersionConstraint
+from univers.version_range import EbuildVersionRange
+from univers.versions import GentooVersion
+from univers.versions import InvalidVersion
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackageV2
+from vulnerabilities.importer import ReferenceV2
+from vulnerabilities.importer import VulnerabilitySeverity
+from vulnerabilities.management.commands.commit_export import logger
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+from vulnerabilities.severity_systems import GENERIC
+
+
+class GentooImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
+    repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git"
+    spdx_license_expression = "CC-BY-SA-4.0"
+    # the license notice is at this url https://anongit.gentoo.org/ says:
+    # The contents of this document, unless otherwise expressly stated, are licensed
+    # under the [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/) license.
+    license_url = "https://creativecommons.org/licenses/by-sa/4.0/"
+    pipeline_id = "gentoo_importer_v2"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.clone,
+            cls.collect_and_store_advisories,
+            cls.clean_downloads,
+        )
+
+    def clone(self):
+        self.log(f"Cloning `{self.repo_url}`")
+        self.vcs_response = fetch_via_vcs(self.repo_url)
+
+    def advisories_count(self):
+        advisory_dir = Path(self.vcs_response.dest_dir)
+        return sum(1 for _ in advisory_dir.rglob("*.xml"))
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        base_path = Path(self.vcs_response.dest_dir)
+        for file_path in base_path.glob("**/*.xml"):
+            yield from self.process_file(file_path)
+
+    def process_file(self, file):
+        cves = []
+        summary = ""
+        xml_root = ET.parse(file).getroot()
+        id = xml_root.attrib.get("id")
+        glsa = "GLSA-" + id
+
+        vuln_references = [
+            ReferenceV2(
+                reference_id=glsa,
+                url=f"https://security.gentoo.org/glsa/{id}",
+            )
+        ]
+
+        severities = []
+        affected_packages = []
+        for child in xml_root:
+            if child.tag == "references":
+                cves = self.cves_from_reference(child)
+
+            if child.tag == "synopsis":
+                summary = child.text
+
+            if child.tag == "affected":
+                affected_packages = list(affected_and_safe_purls(child))
+
+            if child.tag == "impact":
+                severity_value = child.attrib.get("type")
+                if severity_value:
+                    severities.append(VulnerabilitySeverity(system=GENERIC, value=severity_value))
+
+        yield AdvisoryData(
+            advisory_id=glsa,
+            aliases=cves,
+            summary=summary,
+            references_v2=vuln_references,
+            severities=severities,
+            affected_packages=affected_packages,
+            url=f"https://security.gentoo.org/glsa/{id}"
+            if id
+            else "https://security.gentoo.org/glsa",
+            original_advisory_text=file,
+        )
+
+    def clean_downloads(self):
+        if self.vcs_response:
+            self.log("Removing cloned repository")
+            self.vcs_response.delete()
+
+    def on_failure(self):
+        self.clean_downloads()
+
+    @staticmethod
+    def cves_from_reference(reference):
+        cves = []
+        for child in reference:
+            txt = child.text.strip()
+            match = re.match(r"CVE-\d{4}-\d{4,}", txt)
+            if match:
+                cves.append(match.group())
+        return cves
+
+
+def affected_and_safe_purls(affected_elem):
+    constraints = []
+    for pkg in affected_elem:
+        name = pkg.attrib.get("name")
+        if not name:
+            continue
+        pkg_ns, _, pkg_name = name.rpartition("/")
+        purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns)
+        safe_versions, affected_versions = get_safe_and_affected_versions(pkg)
+
+        for version in safe_versions:
+            try:
+                constraints.append(
+                    VersionConstraint(version=GentooVersion(version), comparator="=").invert()
+                )
+            except InvalidVersion as e:
+                logger.error(f"InvalidVersion - version: {version} - error:{e}")
+
+        for version in affected_versions:
+            try:
+                constraints.append(
+                    VersionConstraint(version=GentooVersion(version), comparator="=")
+                )
+            except InvalidVersion as e:
+                logger.error(f"InvalidVersion - version: {version} - error:{e}")
+
+        if not constraints:
+            continue
+
+        yield AffectedPackageV2(
+            package=purl,
+            affected_version_range=EbuildVersionRange(constraints=constraints),
+            fixed_version_range=None,
+        )
+
+
+def get_safe_and_affected_versions(pkg):
+    # TODO : Revisit why we are skipping some versions in gentoo importer
+    skip_versions = {"1.3*", "7.3*", "7.4*"}
+    safe_versions = set()
+    affected_versions = set()
+    for info in pkg:
+        if info.text in skip_versions:
+            continue
+
+        if info.attrib.get("range"):
+            if len(info.attrib.get("range")) > 2:
+                continue
+
+        if info.tag == "unaffected":
+            # quick hack, to know whether this
+            # version lies in this range, 'e' stands for
+            # equal, which is paired with 'greater' or 'less'.
+            # All possible values of info.attrib['range'] =
+            # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of
+            # which ('rle', 'rge', 'rgt') are ignored, because they compare
+            # 'release' not the 'version'.
+            if "e" in info.attrib["range"]:
+                safe_versions.add(info.text)
+            else:
+                affected_versions.add(info.text)
+
+        elif info.tag == "vulnerable":
+            if "e" in info.attrib["range"]:
+                affected_versions.add(info.text)
+            else:
+                safe_versions.add(info.text)
+
+    return safe_versions, affected_versions

vulnerabilities/tests/pipelines/v2_importers/test_gentoo_importer_v2.py

@@ -0,0 +1,38 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from pathlib import Path
+from unittest.mock import Mock
+from unittest.mock import patch
+
+import pytest
+
+from vulnerabilities.pipelines.v2_importers.gentoo_importer import GentooImporterPipeline
+from vulnerabilities.tests import util_tests
+
+TEST_DATA = Path(__file__).parent.parent.parent / "test_data" / "gentoo_v2"
+
+TEST_CVE_FILES = [
+    TEST_DATA / "glsa-201709-09.xml",
+    TEST_DATA / "glsa-202511-02.xml",
+    TEST_DATA / "glsa-202512-01.xml",
+]
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize("xml_file", TEST_CVE_FILES)
+def test_gentoo_advisories_per_file(xml_file):
+    pipeline = GentooImporterPipeline()
+    pipeline.vcs_response = Mock(dest_dir=TEST_DATA)
+
+    with patch.object(Path, "glob", return_value=[xml_file]):
+        result = [adv.to_dict() for adv in pipeline.collect_advisories()]
+
+    expected_file = xml_file.with_name(xml_file.stem + "-expected.json")
+    util_tests.check_results_against_json(result, expected_file)

vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09-expected.json

@@ -0,0 +1,43 @@
+[
+  {
+    "advisory_id": "GLSA-201709-09",
+    "aliases": [
+      "CVE-2017-9800"
+    ],
+    "summary": "A command injection vulnerability in Subversion may allow remote\n    attackers to execute arbitrary code.",
+    "affected_packages": [
+      {
+        "package": {
+          "type": "ebuild",
+          "namespace": "dev-vcs",
+          "name": "subversion",
+          "version": "",
+          "qualifiers": "",
+          "subpath": ""
+        },
+        "affected_version_range": "vers:ebuild/0.1.1|!=1.9.7",
+        "fixed_version_range": null,
+        "introduced_by_commit_patches": [],
+        "fixed_by_commit_patches": []
+      }
+    ],
+    "references_v2": [
+      {
+        "reference_id": "GLSA-201709-09",
+        "reference_type": "",
+        "url": "https://security.gentoo.org/glsa/201709-09"
+      }
+    ],
+    "patches": [],
+    "severities": [
+      {
+        "system": "generic_textual",
+        "value": "normal",
+        "scoring_elements": ""
+      }
+    ],
+    "date_published": null,
+    "weaknesses": [],
+    "url": "https://security.gentoo.org/glsa/201709-09"
+  }
+]

vulnerabilities/tests/test_data/gentoo_v2/glsa-201709-09.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE glsa SYSTEM "http://www.gentoo.org/dtd/glsa.dtd">
+<glsa id="201709-09">
+  <title>Subversion: Arbitrary code execution</title>
+  <synopsis>A command injection vulnerability in Subversion may allow remote
+    attackers to execute arbitrary code.
+  </synopsis>
+  <product type="ebuild">subversion</product>
+  <announced>2017-09-17</announced>
+  <revised count="1">2017-09-17</revised>
+  <bug>627480</bug>
+  <access>remote</access>
+  <affected>
+    <package name="dev-vcs/subversion" auto="yes" arch="*">
+      <unaffected range="ge">1.9.7</unaffected>
+      <unaffected range="rgt">1.8.18</unaffected>
+      <vulnerable range="lt">1.9.7</vulnerable>
+      <vulnerable range="eq">0.1.1</vulnerable>      
+
+    </package>
+  </affected>
+  <background>
+    <p>Subversion is a version control system intended to eventually replace
+      CVS. Like CVS, it has an optional client-server architecture (where the
+      server can be an Apache server running mod_svn, or an ssh program as in
+      CVS’s :ext: method). In addition to supporting the features found in
+      CVS, Subversion also provides support for moving and copying files and
+      directories.
+    </p>
+  </background>
+  <description>
+    <p>Specially crafted ‘ssh://...’ URLs may allow the owner of the
+      repository to execute arbitrary commands on client’s machine if those
+      commands are already installed on the client’s system. This is
+      especially dangerous when the third-party repository has one or more
+      submodules with specially crafted ‘ssh://...’ URLs. Each time the
+      repository is recursively cloned or submodules are updated the payload
+      will be triggered.
+    </p>
+  </description>
+  <impact type="normal">
+    <p>A remote attacker, by enticing a user to clone a specially crafted
+      repository, could possibly execute arbitrary code with the privileges of
+      the process.
+    </p>
+  </impact>
+  <workaround>
+    <p>There are several alternative ways to fix this vulnerability. Please
+      refer to Subversion Team Announce for more details.
+    </p>
+  </workaround>
+  <resolution>
+    <p>All Subversion 1.9.x users should upgrade to the latest version:</p>
+
+    <code>
+      # emerge --sync
+      # emerge --ask --oneshot --verbose "&gt;=dev-vcs/subversion-1.9.7"
+    </code>
+
+    <p>All Subversion 1.8.x users should upgrade to the latest version:</p>
+
+    <code>
+      # emerge --sync
+      # emerge --ask --oneshot --verbose "&gt;=dev-vcs/subversion-1.8.18"
+    </code>
+  </resolution>
+  <references>
+    <uri link="https://nvd.nist.gov/nvd.cfm?cvename=CVE-2017-9800">
+      CVE-2017-9800
+    </uri>
+    <uri link="https://subversion.apache.org/security/CVE-2017-9800-advisory.txt">
+      Subversion Team Announce
+    </uri>
+  </references>
+  <metadata tag="requester" timestamp="2017-09-01T12:55:21Z">b-man</metadata>
+  <metadata tag="submitter" timestamp="2017-09-17T15:50:43Z">chrisadr</metadata>
+</glsa>