diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 020cf0172..be89a5973 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -9,7 +9,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: [3.8] + python-version: [3.9] steps: - name: Checkout code diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36cdb4862..4428993e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release-aboutcode-hashid.yml b/.github/workflows/pypi-release-aboutcode-hashid.yml new file mode 100644 index 000000000..1260afb31 --- /dev/null +++ b/.github/workflows/pypi-release-aboutcode-hashid.yml @@ -0,0 +1,38 @@ +name: Build aboutcode.hashid Python distributions and publish on PyPI + +on: + workflow_dispatch: + push: + tags: + - "aboutcode.hashid/*" + +jobs: + build-and-publish: + name: Build and publish library to PyPI + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install flot + run: python -m pip install flot --user + + - name: Build binary wheel and source tarball + run: python -m flot --pyproject pyproject-aboutcode.hashid.toml --sdist --wheel --output-dir dist/ + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN_ABOUTCODE_HASHID }} + + - name: Upload built archives + uses: actions/upload-artifact@v4 + with: + name: pypi_archives + path: dist/* \ No newline at end of file diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 22315ff0e..600b046d4 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -37,7 +37,7 @@ jobs: run: python -m build --sdist --wheel --outdir dist/ - name: Upload built archives - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pypi_archives path: dist/* @@ -51,7 +51,7 @@ jobs: steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist @@ -71,7 +71,7 @@ jobs: steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1835ea943..7f6debf44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,145 @@ Release notes ============= + +Version v35.1.0 +--------------------- + +- Use AboutCode mirror for collecting CISA KEV #1685 +- Do not report ghost package as a fix for vulnerability #1679 +- Add pipeline to sort packages #1686 +- Fix urls for API #1678 + + +Version v35.0.0 +--------------------- + +- Add scores in bulk search V1 API #1675 +- Add improver pipeline to flag ghost packages #644 #917 #1395 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1533 +- Add base pipeline for importers and migrate PyPa importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1559 +- Remove dupe Package.get_non_vulnerable_versions by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1570 +- Import data from GSD #706 by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/787 +- Add curl advisories importer by @ambuj-1211 in https://github.com/aboutcode-org/vulnerablecode/pull/1439 +- Update dependencies by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1590 +- Bump django from 4.2.0 to 4.2.15 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1591 +- Bump cryptography from 42.0.4 to 43.0.1 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1582 +- Bump actions/download-artifact from 3 to 4.1.7 in /.github/workflows by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1581 +- Improve export command by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1571 +- Fix typo in Kev requests import by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1594 +- Prepare for release v34.0.1 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1595 +- Bump upload-artifact to v4 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1596 +- Migrate Npm importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1574 +- Use correct regex for CVE by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1599 +- Migrate Nginx importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1575 +- Migrate GitLab importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1580 +- Migrate GitHub importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1584 +- Migrate NVD importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1587 +- Match affected and fixed-by Packages by @johnmhoran in https://github.com/aboutcode-org/vulnerablecode/pull/1528 +- Add management command to commit exported data by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1600 +- Add support to Exploits model by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1562 +- Fix 500 Server Error with DRF browsable API and resolve blank Swagger API documentation by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1603 +- Release v34.0.2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1604 +- Bump VCIO version by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1605 +- Bump django from 4.2.15 to 4.2.16 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1608 +- Bump fetchcode from v0.3.0 to v0.6.0 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1607 +- Use 4-tier system for storing package metadata by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1609 +- Fix vers range crash by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1598 +- Add GitHub action to publish aboutcode.hashid PyPI by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1615 +- Segregate PackageRelatedVulnerability model to new models by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1612 +- Add documentation for new pipeline design by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1621 +- Fix 500 error in /api/cpes endpoint by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1629 +- Migrate pysec importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1628 +- Avoid memory exhaustion during data migration by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1630 +- Add support for Calculating Risk in VulnerableCode by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1593 +- Bulk create in migrations by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1640 +- Update README.rst by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1641 +- Prepare for release v34.1.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1642 +- Add V2 API endpoints by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1631 +- Prepare for release v34.2.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1647 +- Refactor severity score model and fix incorrect suse scores by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1636 +- Add bulk search in v2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1649 +- Prepare release v34.3.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1652 +- Add `on_failure` to handle cleanup during pipeline failure by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1651 +- Fix API bug by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1654 +- Add reference score to package endpoint by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1655 +- Prepare for release v34.3.2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1656 +- Add support for storing exploitability and weighted severity by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1646 +- Avoid migrations on version bumps by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1660 +- Prepare v35.0.0rc1 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1664 + + + +Version v35.0.0rc1 +--------------------- + +- Add support for storing exploitability and weighted severity #1646 +- Avoid migrations on version bumps #1660 + + +Version v34.3.2 +---------------- + +- HOTFIX: Add reference score to package endpoint #1655 + + +Version v34.3.1 +---------------- + +- HOTFIX: Fix API bug #1654 + + +Version v34.3.0 +----------------- + +- Add bulk search in v2 #1649 +- Refactor severity score model and fix incorrect suse scores #1636 + + +Version v34.2.0 +------------------- + +- Add V2 API endpoints #1631 + + +Version v34.1.0 +------------------- + +- Add support for Calculating Package Vulnerability Risk #1593 +- Migrate pysec importer to aboutcode pipeline #1628 +- Fix 500 error in /api/cpes endpoint #1629 +- Add documentation for new pipeline design #1621 +- Segregate PackageRelatedVulnerability model to new models #1612 +- Add GitHub action to publish aboutcode.hashid PyPI #1615 +- Fix vers range crash #1598 +- Use 4-tier system for storing package metadata #1609 + + +Version v34.0.2 +------------------- + +- Add management command to commit exported vulnerability data (#1600) +- Fix API 500 error (#1603) + + +Version v34.0.1 +------------------- + +- Add Pipeline to flag ghost packages (#1533) +- Add logging configuration (#1533) +- Drop support for python 3.8 (#1533) +- Drop using docker-compose and use the built-in "docker compose" instead +- Upgrade core dependencies including Django and Rest Framework +- Fix typo in KEV improver (#1594) + + +Version v34.0.0 +------------------- + +- Improve API performance. +- Add severity range score in API. +- Refactor GitlabDataSource to work with browser extension + + Version v34.0.0rc5 ------------------- diff --git a/Makefile b/Makefile index b745c5704..067cb419f 100644 --- a/Makefile +++ b/Makefile @@ -125,13 +125,13 @@ bump: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ docker-images: @echo "-> Build Docker services" - docker-compose build + docker compose build @echo "-> Pull service images" - docker-compose pull + docker compose pull @echo "-> Save the service images to a compressed tar archive in the dist/ directory" @mkdir -p dist/ @docker save postgres vulnerablecode_vulnerablecode nginx | gzip > dist/vulnerablecode-images-`git describe --tags`.tar.gz diff --git a/README.rst b/README.rst index a5a256b13..12be81e57 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,6 @@ we are trying to change this and evolve the status quo in a few other areas! Package URL themselves were designed first in ScanCode and VulnerableCode and are now a de-facto standard for vulnerability management and package references. - See https://github.com/package-url/purl-spec The VulnerableCode project is a FOSS community resource to help improve the @@ -47,87 +46,86 @@ security of the open source software ecosystem and its users at large. VulnerableCode consists of a database and the tools to collect, refine and keep the database current. -.. warning:: - VulnerableCode is under active development and is not yet fully - usable. -Read more about VulnerableCode https://vulnerablecode.readthedocs.org/ +.. pull-quote:: + **Warning** + + VulnerableCode is under active development and is not yet fully + usable. + -VulnerableCode is financially supported by NLnet, nexB, Google (through the -GSoC) and the active contributions of several volunteers. +Read more about VulnerableCode https://vulnerablecode.readthedocs.org/ VulnerableCode tech stack is Python, Django, PostgreSQL, nginx and Docker and several libraries. Getting started ---------------- +=============== Run with Docker -^^^^^^^^^^^^^^^^ +--------------- -First install docker and docker-compose, then run:: +First install docker, then run + +.. code:: bash git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode make envfile - docker-compose build - docker-compose up -d - docker-compose run vulnerablecode ./manage.py import --list + docker compose build + docker compose up -d + docker compose run vulnerablecode ./manage.py import --list + +Then run an importer for nginx advisories (which is small) -Then run an importer for nginx advisories (which is small):: +.. code:: bash - docker-compose exec vulnerablecode ./manage.py import vulnerabilities.importers.nginx.NginxImporter - docker-compose exec vulnerablecode ./manage.py improve --all + docker compose exec vulnerablecode ./manage.py import nginx_importer + docker compose exec vulnerablecode ./manage.py improve --all At this point, the VulnerableCode app and API should be up and running with some data at http://localhost Populate VulnerableCode database -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------- VulnerableCode data collection works in two steps: importing data from multiple sources and then refining and improving how package and software vulnerabilities are related. -To run all importers and improvers use this:: +To run all importers and improvers use this + +.. code:: bash ./manage.py import --all + +.. code:: bash + ./manage.py improve --all Local development installation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ + +On a Debian system, use this -On a Debian system, use this:: +.. code:: bash sudo apt-get install python3-venv python3-dev postgresql libpq-dev build-essential git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode make dev envfile postgres make test source venv/bin/activate - ./manage.py import vulnerabilities.importers.nginx.NginxImporter + ./manage.py import nginx_importer ./manage.py improve --all make run At this point, the VulnerableCode app and API is up at http://127.0.0.1:8001/ -Interface -^^^^^^^^^^ - - -VulnerableCode comes with a minimal web UI: - -.. image:: vulnerablecode-ui.png - -And a JSON API and its minimal web documentation: - -.. image:: vulnerablecode-json-api.png -.. image:: vulnerablecode-api-doc.png - License -^^^^^^^^^^ +======== Copyright (c) nexB Inc. and others. All rights reserved. @@ -147,19 +145,137 @@ See https://github.com/nexB/vulnerablecode for support or download. See https://aboutcode.org for more information about nexB OSS projects. -Acknowledgements -^^^^^^^^^^^^^^^^ -This project was funded through the NGI0 PET Fund, a fund established by -NLnet with financial support from the European Commission's Next Generation -Internet programme, under the aegis of DG Communications Networks, Content -and Technology under grant agreement No 825310. +Acknowledgements, Funding, Support and Sponsoring +================================================= + +This project is funded, supported and sponsored by: + +- Generous support and contributions from users like you! +- the European Commission NGI programme +- the NLnet Foundation +- the Swiss State Secretariat for Education, Research and Innovation (SERI) +- Google, including the Google Summer of Code and the Google Seasons of Doc programmes +- Mercedes-Benz Group +- Microsoft and Microsoft Azure +- AboutCode ASBL +- nexB Inc. + + + +|europa| |dgconnect| + +|ngi| |nlnet| + +|aboutcode| |nexb| + + + +This project was funded through the NGI0 PET Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 825310. + +|ngizeropet| https://nlnet.nl/project/VulnerableCode/ + + +This project was funded through the NGI0 Discovery Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 825322. + +|ngidiscovery| https://nlnet.nl/project/vulnerabilitydatabase/ + + +This project was funded through the NGI0 Core Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101092990. + +|ngizerocore| https://nlnet.nl/project/VulnerableCode-enhancements/ + + +This project is funded through the NGI0 Entrust Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101069594. + +|ngizeroentrust| https://nlnet.nl/project/FederatedSoftwareMetadata/ + + +This project was funded through the NGI0 Commons Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101135429. Additional +funding is made available by the Swiss State Secretariat for Education, Research and Innovation +(SERI). + +|ngizerocommons| |swiss| https://nlnet.nl/project/FederatedCodeNext/ + +This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101069594. + +|ngizeroentrust| https://nlnet.nl/project/CRAVEX/ + + + +.. |nlnet| image:: https://nlnet.nl/logo/banner.png + :target: https://nlnet.nl + :height: 50 + :alt: NLnet foundation logo + +.. |ngi| image:: https://ngi.eu/wp-content/uploads/thegem-logos/logo_8269bc6efcf731d34b6385775d76511d_1x.png + :target: https://ngi.eu35 + :height: 50 + :alt: NGI logo + +.. |nexb| image:: https://nexb.com/wp-content/uploads/2022/04/nexB.svg + :target: https://nexb.com + :height: 30 + :alt: nexB logo + +.. |europa| image:: https://ngi.eu/wp-content/uploads/sites/77/2017/10/bandiera_stelle.png + :target: http://ec.europa.eu/index_en.htm + :height: 40 + :alt: Europa logo + +.. |aboutcode| image:: https://aboutcode.org/wp-content/uploads/2023/10/AboutCode.svg + :target: https://aboutcode.org/ + :height: 30 + :alt: AboutCode logo + +.. |swiss| image:: https://www.sbfi.admin.ch/sbfi/en/_jcr_content/logo/image.imagespooler.png/1493119032540/logo.png + :target: https://www.sbfi.admin.ch/sbfi/en/home/seri/seri.html + :height: 40 + :alt: Swiss logo + +.. |dgconnect| image:: https://commission.europa.eu/themes/contrib/oe_theme/dist/ec/images/logo/positive/logo-ec--en.svg + :target: https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en + :height: 40 + :alt: EC DG Connect logo + +.. |ngizerocore| image:: https://nlnet.nl/image/logos/NGI0_tag.svg + :target: https://nlnet.nl/core + :height: 40 + :alt: NGI Zero Core Logo + +.. |ngizerocommons| image:: https://nlnet.nl/image/logos/NGI0_tag.svg + :target: https://nlnet.nl/commonsfund/ + :height: 40 + :alt: NGI Zero Commons Logo + +.. |ngizeropet| image:: https://nlnet.nl/image/logos/NGI0PET_tag.svg + :target: https://nlnet.nl/PET + :height: 40 + :alt: NGI Zero PET logo -https://nlnet.nl/project/VulnerableCode/ +.. |ngizeroentrust| image:: https://nlnet.nl/image/logos/NGI0Entrust_tag.svg + :target: https://nlnet.nl/entrust + :height: 38 + :alt: NGI Zero Entrust logo -This project was funded through the NGI0 Discovery Fund, a fund established -by NLnet with financial support from the European Commission's Next Generation -Internet programme, under the aegis of DG Communications Networks, Content -and Technology under grant agreement No 825322. +.. |ngiassure| image:: https://nlnet.nl/image/logos/NGIAssure_tag.svg + :target: https://nlnet.nl/image/logos/NGIAssure_tag.svg + :height: 32 + :alt: NGI Assure logo -https://nlnet.nl/project/vulnerabilitydatabase/ +.. |ngidiscovery| image:: https://nlnet.nl/image/logos/NGI0Discovery_tag.svg + :target: https://nlnet.nl/discovery/ + :height: 40 + :alt: NGI Discovery logo diff --git a/aboutcode/hashid/CHANGELOG.rst b/aboutcode/hashid/CHANGELOG.rst new file mode 100644 index 000000000..2d1f39adf --- /dev/null +++ b/aboutcode/hashid/CHANGELOG.rst @@ -0,0 +1,13 @@ +Changelog +============= + + +v0.2.0 (December 05, 2024) +--------------------------- + +- Use 4-tier system for storing package metadata https://github.com/aboutcode-org/vulnerablecode/pull/1609 + +v0.1.0 (September 12, 2024) +--------------------------- + +- Initial release of the ``aboutcode.hashid`` library. \ No newline at end of file diff --git a/aboutcode/hashid/README.rst b/aboutcode/hashid/README.rst new file mode 100644 index 000000000..9c67387af --- /dev/null +++ b/aboutcode/hashid/README.rst @@ -0,0 +1,15 @@ +aboutcode.hashid +================== + +This is a library of utilities to compute ids and file paths for AboutCode using VCID and PURLs. + +License +------- + +Copyright (c) nexB Inc. and others. All rights reserved. + +SPDX-License-Identifier: Apache-2.0 + +See https://github.com/aboutcode-org/vulnerablecode for support or download. + +See https://aboutcode.org for more information about AboutCode OSS projects. diff --git a/aboutcode/hashid/__init__.py b/aboutcode/hashid/__init__.py new file mode 100644 index 000000000..0d80ffafd --- /dev/null +++ b/aboutcode/hashid/__init__.py @@ -0,0 +1,410 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# Portions Copyright (c) The Python Software Foundation +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 and Python-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from hashlib import sha256 +from math import ceil +from pathlib import Path +from typing import Union +from urllib.parse import quote +from uuid import uuid4 + +from packageurl import PackageURL +from packageurl import normalize_qualifiers +from packageurl import normalize_subpath + +__version__ = "0.2.0" + + +""" +General purpose utilities to create Vulnerability Ids aka. VCID and content-defined, hash-based +paths to store Vulnerability and Package data using these paths in many balanced directories. + +The reason why this is needed is to store many vulnerability and package metadata files, we need +to distribute these files in multiple directories and avoid too many files in the same directory +which makes every filesystem performance suffer. + +In addition, when storing these files in Git repositories, we need to avoid creating any repository +with too many files that would make using this repository impractical or exceed the limits of some +repository hosting services. + +Therefore we are storing vulnerability data using a directory tree using the first few characters +of the PURL hash of a package or the UUID of a vulnerability id. +""" + +VULNERABILITY_REPO_NAME = "aboutcode-vulnerabilities" + +PACKAGE_REPOS_NAME_PREFIX = "aboutcode-packages" +PURLS_FILENAME = "purls.yml" +VULNERABILITIES_FILENAME = "vulnerabilities.yml" + + +def build_vcid(prefix="VCID"): + """ + Return a new Vulnerable Code ID (aka. VCID) which is a strongly unique vulnerability + identifier string using the provided ``prefix``. A VCID is composed of a four letter prefix, and + three segments composed of four letters and digits each separated by a dash. + For example:: + >>> import re + >>> vcid = build_vcid() + >>> assert re.match('VCID(-[a-hjkm-z1-9]{4}){3}', vcid), vcid + + We were mistakenly not using enough bits. The symptom was that the last + segment of the VCID was always string with "aaa" This ensure we are now OK: + >>> vcids = [build_vcid() for _ in range(50)] + >>> assert not any(vid.split("-")[-1].startswith("aaa") for vid in vcids) + """ + uid = uuid4().bytes + # we keep three segments of 4 base32-encoded bytes, 3*4=12 + # which corresponds to 60 bits + # because each base32 byte can store 5 bits (2**5 = 32) + uid = base32_custom(uid)[:12].decode("utf-8").lower() + return f"{prefix}-{uid[:4]}-{uid[4:8]}-{uid[8:12]}" + + +def get_vcid_yml_file_path(vcid: str): + """ + Return the path to the vulnerability YAML file for a VCID. + """ + return Path(VULNERABILITY_REPO_NAME) / vulnerability_yml_path(vcid) + + +# This custom 32 characters alphabet is designed to avoid visually easily confusable characters: +# i and l +# 0 and o +_base32_alphabet = b"abcdefghjkmnpqrstuvwxyz123456789" +_b32tab = [bytes((i,)) for i in _base32_alphabet] +_base32_table = [a + b for a in _b32tab for b in _b32tab] + +base32_custom_alphabet = _base32_alphabet.decode("utf-8") + + +def base32_custom(btes): + """ + Encode the ``btes`` bytes using a custom Base32 encoding with a custom alphabet and return a + lowercase byte string. This alphabet is designed to avoid confusable characters. + + Not meant for general purpose Base32 encoding as this is not designed to ever be decoded. + Code copied and modified from the Python Standard Library: base64._b32encode function + + For example:: + >>> base32_custom(b'abcd') + b'abtze25e' + + >>> base32_custom(b'abcde00000xxxxxPPPPP') + b'pfugg3dfga2dapbtsb6ht8d2mbjfaxct' + """ + + encoded = bytearray() + from_bytes = int.from_bytes + + for i in range(0, len(btes), 5): + c = from_bytes(btes[i : i + 5], "big") # big-endian + encoded += ( + _base32_table[c >> 30] # bits 1 - 10 + + _base32_table[(c >> 20) & 0x3FF] # bits 11 - 20 + + _base32_table[(c >> 10) & 0x3FF] # bits 21 - 30 + + _base32_table[c & 0x3FF] # bits 31 - 40 + ) + return bytes(encoded) + + +def vulnerability_yml_path(vcid): + """ + Return the path to a vulnerability YAML file crafted from the ``vcid`` VCID vulnerability id. + + The approach is to distribute the files in many directories to avoid having too many files in + any directory and be able to find the path to a vulnerability file given its VCID distributed on + the first two characters of the UUID section of a VCID. + + The UUID is using a base32 encoding, hence keeping two characters means 32 x 32 = 1024 + possibilities, meaning 1024 directories. Given a current count of vulnerabilities of about 300K, + mid 2024 this gives ample distribution of about 1000 vulnerabilities in each of 1000 directories + and plenty of room to grow. + + The serialized vulnerability data should about 300MB compressed and should be storable in single + Git repository. + + For example:: + >>> vulnerability_yml_path("VCID-s9bw-m429-aaaf") + 's9/VCID-s9bw-m429-aaaf.yml' + """ + prefix = vcid[5 : 5 + 2] + return f"{prefix}/{vcid}.yml" + + +def get_package_base_dir(purl: Union[PackageURL, str]): + """ + Return the base path to a Package directory (ignoring version) for a purl + """ + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + + path_elements = package_path_elements(purl) + phash, core_path, _pversion, _extra_path = path_elements + return Path(f"{PACKAGE_REPOS_NAME_PREFIX}-{purl.type}-{phash}") / core_path + + +def get_package_purls_yml_file_path(purl: Union[PackageURL, str]): + """ + Return the path to a Package purls.yml YAML for a purl. + """ + return get_package_base_dir(purl) / PURLS_FILENAME + + +def get_package_vulnerabilities_yml_file_path(purl: Union[PackageURL, str]): + """ + Return the path to a Package vulnerabilities.yml YAML for a purl. + """ + return get_package_base_dir(purl) / VULNERABILITIES_FILENAME + + +# We use a 4-tier system for storing package metadata. +# The tiers are as follows: +# 1. Super Large Ecosystem (~5M packages): 2^10 = 1,024 git repositories +# 2. Large Ecosystem (~500K packages): 2^7 = 128 git repositories +# 3. Medium Ecosystem (~50K packages): 2^5 = 32 git repositories +# 4. Small Ecosystem (~2K packages): 2^0 = 1 git repository +# See https://github.com/aboutcode-org/federatedcode/issues/3#issuecomment-2388371726 +BIT_COUNT_BY_ECOSYSTEM = { + # Super Large Ecosystem + "github": 10, + "npm": 10, + # Large Ecosystem + "golang": 7, + "maven": 7, + "nuget": 7, + "perl": 7, + "php": 7, + "pypi": 7, + "ruby": 7, + # Medium Ecosystem + "alpm": 5, + "bitbucket": 5, + "cocoapods": 5, + "composer": 5, + "deb": 5, + "docker": 5, + "gem": 5, + "generic": 5, + "huggingface": 5, + "mlflow": 5, + "pub": 5, + "rpm": 5, + # Small Ecosystem + "bitnami": 0, + "cargo": 0, + "conan": 0, + "conda": 0, + "cpan": 0, + "cran": 0, + "hackage": 0, + "hex": 0, + "luarocks": 0, + "swift": 0, +} + + +def package_path_elements(purl: Union[PackageURL, str]): + """ + Return 4-tuple of POSIX path strings crafted from the ``purl`` package PURL string or object. + The tuple members are: (purl_hash, core_path, purl.version, extra_path) + These members can be joined using a POSIX "/" path separator to store package data distributed + evenly in many directories, where package data of the same package is co-located in the same + root directory. + + The approach is to distribute the files in many directories to avoid having too many data files + in any directory and be able to find the path to the YAML data files for a package given its + PURL. For this we use the first characters of the "purl hash" to construct a path. + + A purl hash has 8,192 possible values, meaning 8,192 directories or repositories, basically used + as a hash table. Given an estimated count of packages of about 30 million in mid 2024, this + gives ample distribution of about 4,000 packages in each of these top level directories and some + room to grow. + + The size to store compressed package metadata is guesstimated to be 1MB on average and 10MB for + a full scan. This means that each directory will store 4K * 10MB ~= 4 GB. This should keep + backing git repositories to a reasonable size, below 5GB. + + The storage scheme is designed to create this path structure: + + : top level directory or repository + // : sub directories + purls.yml : YAML file with known versions for this package ordered from oldest to newest + vulnerabilities.yml : YAML file with known vulnerabilities affecting (and fixed by) this package + + : one sub directory for each version + metadata.yml : ABOUT YAML file with package origin and license metadata for this version + scancode-scan.yml : a scancode scan for this package version + foo-scan.yml : a scan for this package version created with tool foo + sbom.cdx.1.4.json : a CycloneDX SBOM + sbom.cdx.1.5.json : a CycloneDX SBOM + sbom.spdx.2.2.json : a SPDX SBOM + .... other files + + : one sub directory for each quote-encoded if any + metadata.yml : ABOUT YAML file with package origin and license metadata for this version + scancode-scan.yml : a scancode scan for this package version + foo-scan.yml : a scan for this package version created with tool foo + sbom.cdx.1.4.json : a CycloneDX SBOM + ... other files + + Some examples: + + We keep the same prefix for different versions:: + + >>> package_path_elements("pkg:pypi/license_expression@30.3.1") + ('50', 'pypi/license-expression', '30.3.1', '') + >>> package_path_elements("pkg:pypi/license_expression@10.3.1") + ('50', 'pypi/license-expression', '10.3.1', '') + + We encode with quotes, avoid double encoding of already quoted parts to make subpaths easier + for filesystems:: + + >>> package_path_elements("pkg:pypi/license_expression@30.3.1?foo=bar&baz=bar#sub/path") + ('50', 'pypi/license-expression', '30.3.1', 'baz%3Dbar%26foo%3Dbar%23sub%2Fpath') + + >>> purl = PackageURL( + ... type="pypi", + ... name="license_expression", + ... version="b#ar/?30.3.2!", + ... qualifiers=dict(foo="bar"), + ... subpath="a/b/c") + >>> package_path_elements(purl) + ('50', 'pypi/license-expression', 'b%23ar%2F%3F30.3.2%21', 'foo%3Dbar%23a%2Fb%2Fc') + """ + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + + bit_count = BIT_COUNT_BY_ECOSYSTEM.get(purl.type, 0) + purl_hash = get_purl_hash(purl=purl, _bit_count=bit_count) + + if ns := purl.namespace: + ns_name = f"{ns}/{purl.name}" + else: + ns_name = purl.name + + extra_path = "" + if pq := purl.qualifiers: + # note that we percent-quote everything including the / character + extra_path = quote_more(normalize_qualifiers(pq, encode=True)) + if psp := purl.subpath: + psp = normalize_subpath(psp, encode=True) + extra_path += quote_more(f"#{psp}") + + core_path = f"{purl.type}/{ns_name}" + + return purl_hash, core_path, quote_more(purl.version), extra_path + + +def quote_more(qs): + """ + Return a quoted string from ``qs`` string by quoting all non-quoted characters ignoring already + quoted characters. This makes the quoted string safer to use in a path. + + For example:: + >>> quote_more("foo") + 'foo' + + >>> quote_more("foo/bar") + 'foo%2Fbar' + + >>> quote_more("foo%2Fbar") + 'foo%2Fbar' + """ + if not qs: + return qs + try: + return quote(qs, safe="%") + except Exception as e: + raise Exception(f"Failed to quote_more: {qs!r}") from e + + +def get_core_purl(purl: Union[PackageURL, str]): + """ + Return a new "core" purl from a ``purl`` object, dropping version, qualifiers and subpath. + """ + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + + purld = purl.to_dict() + del purld["version"] + del purld["qualifiers"] + del purld["subpath"] + return PackageURL(**purld) + + +def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 0) -> str: + """ + Return a short lower cased hash string from a ``purl`` string or object. The PURL is normalized + and we drop its version, qualifiers and subpath. + + This function takes a normalized PURL string and a ``_bit_count`` argument defaulting to 0 bits + which represents 2**0 = 1 possible hash value. It returns a fixed length short hash string + that is left-padded with zeros. + + The hash length is derived from the bit_count and the number of bits-per-byte stored in an hex + encoding of this bits count. For 10 bits, this means up to 3 characters. + + The function is carefully designed to be portable across tech stacks and easy to implement in + many programming languages: + + - the hash is computed using sha256 which is available is all common language, + - the hash is using simple lowercased HEX encoding, + - we use simple arithmetics on integer with modulo. + + The processing goes through these steps: + + First, a SHA256 hash computed on the PURL bytes encoded as UTF-8. + + Then, the hash digest bytes are converted to an integer, which is reduced modulo the largest + possible value for the bit_count. + + Finally, this number is converted to hex, left-padded with zero up to the hash_length, and + returned as a lowercase string. + + For example:: + + The hash does not change with version or qualifiers:: + >>> get_purl_hash("pkg:pypi/univers@30.12.0", 7) + '09' + >>> get_purl_hash("pkg:pypi/univers@10.12.0", 7) + '09' + >>> get_purl_hash("pkg:pypi/univers@30.12.0?foo=bar#sub/path", 7) + '09' + + The hash is left padded with zero if it:: + >>> get_purl_hash("pkg:pypi/expressionss", 7) + '57' + + We normalize the PURL. Here pypi normalization always uses dash for underscore :: + + >>> get_purl_hash("pkg:pypi/license_expression", 7) + '50' + >>> get_purl_hash("pkg:pypi/license-expression", 7) + '50' + + Originally from: + https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154 + """ + + core_purl = get_core_purl(purl).to_string() + # compute the hash from a UTF-8 encoded string + purl_bytes = core_purl.encode("utf-8") + hash_bytes = sha256(purl_bytes).digest() + # ... converted to integer so we can truncate with modulo. Note that we use big endian. + hash_int = int.from_bytes(hash_bytes, "big") + # take a modulo based on bit count to truncate digest to the largest int value for the bitcount + max_int = 2**_bit_count + short_hash = hash_int % max_int + # maximum number of hex characters in the hash string + bits_per_hex_byte = 4 + num_chars_in_hash = ceil(_bit_count / bits_per_hex_byte) + # return an hex "x" string left padded with 0 + return f"{short_hash:0{num_chars_in_hash}x}".lower() diff --git a/aboutcode/hashid/__init__.py.ABOUT b/aboutcode/hashid/__init__.py.ABOUT new file mode 100644 index 000000000..2cf71c153 --- /dev/null +++ b/aboutcode/hashid/__init__.py.ABOUT @@ -0,0 +1,7 @@ +about_resource: __init__.py +notes: the base32_custom() function is derived from Python base64.py _b32encode function +download_url: https://github.com/python/cpython/blob/77133f570dcad599e5b1199c39e999bfac959ae2/Lib/base64.py#L164 +purl: pkg:github.com/python/cpython@77133f570dcad599e5b1199c39e999bfac959ae2#/Lib/base64.py +license_expression_spdx: Python-2.0 +license_expression: python +copyright: Copyright (c) The Python Software Foundation \ No newline at end of file diff --git a/aboutcode/hashid/python.LICENSE b/aboutcode/hashid/python.LICENSE new file mode 100644 index 000000000..3b3c2bef7 --- /dev/null +++ b/aboutcode/hashid/python.LICENSE @@ -0,0 +1,192 @@ +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) Python Software Foundation; +All Rights Reserved" are retained in Python alone or in any derivative version +prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 +------------------------------------------- + +BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 + +1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an +office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the +Individual or Organization ("Licensee") accessing and otherwise using +this software in source or binary form and its associated +documentation ("the Software"). + +2. Subject to the terms and conditions of this BeOpen Python License +Agreement, BeOpen hereby grants Licensee a non-exclusive, +royalty-free, world-wide license to reproduce, analyze, test, perform +and/or display publicly, prepare derivative works, distribute, and +otherwise use the Software alone or in any derivative version, +provided, however, that the BeOpen Python License is retained in the +Software, alone or in any derivative version prepared by Licensee. + +3. BeOpen is making the Software available to Licensee on an "AS IS" +basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE +SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS +AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY +DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +5. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +6. This License Agreement shall be governed by and interpreted in all +respects by the law of the State of California, excluding conflict of +law provisions. Nothing in this License Agreement shall be deemed to +create any relationship of agency, partnership, or joint venture +between BeOpen and Licensee. This License Agreement does not grant +permission to use BeOpen trademarks or trade names in a trademark +sense to endorse or promote products or services of Licensee, or any +third party. As an exception, the "BeOpen Python" logos available at +http://www.pythonlabs.com/logos.html may be used according to the +permissions granted on that web page. + +7. By copying, installing or otherwise using the software, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + +CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 +--------------------------------------- + +1. This LICENSE AGREEMENT is between the Corporation for National +Research Initiatives, having an office at 1895 Preston White Drive, +Reston, VA 20191 ("CNRI"), and the Individual or Organization +("Licensee") accessing and otherwise using Python 1.6.1 software in +source or binary form and its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, CNRI +hereby grants Licensee a nonexclusive, royalty-free, world-wide +license to reproduce, analyze, test, perform and/or display publicly, +prepare derivative works, distribute, and otherwise use Python 1.6.1 +alone or in any derivative version, provided, however, that CNRI's +License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) +1995-2001 Corporation for National Research Initiatives; All Rights +Reserved" are retained in Python 1.6.1 alone or in any derivative +version prepared by Licensee. Alternately, in lieu of CNRI's License +Agreement, Licensee may substitute the following text (omitting the +quotes): "Python 1.6.1 is made available subject to the terms and +conditions in CNRI's License Agreement. This Agreement together with +Python 1.6.1 may be located on the Internet using the following +unique, persistent identifier (known as a handle): 1895.22/1013. This +Agreement may also be obtained from a proxy server on the Internet +using the following URL: http://hdl.handle.net/1895.22/1013". + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python 1.6.1 or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python 1.6.1. + +4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" +basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. This License Agreement shall be governed by the federal +intellectual property law of the United States, including without +limitation the federal copyright law, and, to the extent such +U.S. federal law does not apply, by the law of the Commonwealth of +Virginia, excluding Virginia's conflict of law provisions. +Notwithstanding the foregoing, with regard to derivative works based +on Python 1.6.1 that incorporate non-separable material that was +previously distributed under the GNU General Public License (GPL), the +law of the Commonwealth of Virginia shall govern this License +Agreement only as to issues arising under or with respect to +Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this +License Agreement shall be deemed to create any relationship of +agency, partnership, or joint venture between CNRI and Licensee. This +License Agreement does not grant permission to use CNRI trademarks or +trade name in a trademark sense to endorse or promote products or +services of Licensee, or any third party. + +8. By clicking on the "ACCEPT" button where indicated, or by copying, +installing or otherwise using Python 1.6.1, Licensee agrees to be +bound by the terms and conditions of this License Agreement. + + ACCEPT + + +CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 +-------------------------------------------------- + +Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, +The Netherlands. All rights reserved. + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appear in all copies and that +both that copyright notice and this permission notice appear in +supporting documentation, and that the name of Stichting Mathematisch +Centrum or CWI not be used in advertising or publicity pertaining to +distribution of the software without specific, written prior +permission. + +STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO +THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE +FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. \ No newline at end of file diff --git a/aboutcode/hashid/tests/test_hashid.py b/aboutcode/hashid/tests/test_hashid.py new file mode 100644 index 000000000..f83944984 --- /dev/null +++ b/aboutcode/hashid/tests/test_hashid.py @@ -0,0 +1,54 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# Portions Copyright (c) The Python Software Foundation +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 and Python-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest + +from aboutcode.hashid import package_path_elements + + +@pytest.mark.parametrize( + "purl, purl_hash", + [ + ("pkg:maven/org.apache.commons/io", "4f"), + ("pkg:GOLANG/google.golang.org/genproto@abcdedf#/googleapis/api/annotations/", "4a"), + ("pkg:golang/github.com/nats-io/nats-server/v2/server@v1.2.9", "22"), + ("pkg:bitbucket/birKenfeld/pyGments-main@244fd47e07d1014f0aed9c", "03"), + ("pkg:github/Package-url/purl-Spec@244fd47e07d1004f0aed9c", "095"), + ("pkg:deb/debian/curl@7.50.3-1?arch=i386&distro=jessie", "19"), + ( + "pkg:docker/customer/dockerimage@sha256:244fd47e07d1004f0aed9c?repository_url=gcr.io", + "10", + ), + ("pkg:gem/jruby-launcher@1.1.2?Platform=java", "1e"), + ( + "pkg:Maven/org.apache.xmlgraphics/batik-anim@1.9.1?repositorY_url=repo.spring.io/release&classifier=sources", + "28", + ), + ( + "pkg:Maven/org.apache.xmlgraphics/batik-anim@1.9.1?repositorY_url=repo.spring.io/release&extension=pom", + "28", + ), + ("pkg:Maven/net.sf.jacob-project/jacob@1.14.3?type=dll&classifier=x86", "17"), + ("pkg:npm/%40angular/animation@12.3.1", "323"), + ("pkg:Nuget/EnterpriseLibrary.Common@6.0.1304", "63"), + ("pkg:PYPI/Django_package@1.11.1.dev1", "00"), + ("pkg:composer/guzzlehttp/promises@2.0.2", "1d"), + ("pkg:Rpm/fedora/curl@7.50.3-1.fc25?Arch=i386&Distro=fedora-25", "16"), + ("pkg:maven/HTTPClient/HTTPClient@0.3-3", "4d"), + ("pkg:maven/mygroup/myartifact@1.0.0%20Final?mykey=my%20value", "6f"), + ("pkg:npm/@babel/core#/googleapis/api/annotations/", "0dc"), + ("pkg:npm/@babel/core@1.0.2#/googleapis/api/annotations/", "0dc"), + ("pkg:npm/core@1.0.2#/googleapis/api/annotations/", "23b"), + ("pkg:npm/core#/googleapis/api/annotations/", "23b"), + ], +) +def test_purl_hash(purl, purl_hash): + result_hash, *_ = package_path_elements(purl) + assert result_hash == purl_hash diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf10..788b03961 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,6 +5,7 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source BUILDDIR = build @@ -14,6 +15,13 @@ help: .PHONY: help Makefile +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/docs/source/README.gif b/docs/source/README.gif deleted file mode 100644 index 74b7a6512..000000000 Binary files a/docs/source/README.gif and /dev/null differ diff --git a/docs/source/conf.py b/docs/source/conf.py index c0b713034..05cec2924 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,6 +35,7 @@ "https://anongit.gentoo.org/git/data/glsa.git", # Git only link "https://www.softwaretestinghelp.com/how-to-write-good-bug-report/", # Cloudflare protection "https://www.openssl.org/news/vulnerabilities.xml", # OpenSSL legacy advisory URL, not longer available + "https://example.org/api/non-existent-packages", ] # Add any Sphinx extension module names here, as strings. They can be diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index fa6e7075b..ba634ecc8 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -18,9 +18,9 @@ Do Your Homework ---------------- Before adding a contribution or create a new issue, take a look at the project’s -`README `_, read through our +`README `_, read through our `documentation `_, -and browse existing `issues `_, +and browse existing `issues `_, to develop some understanding of the project and confirm whether a given issue/feature has previously been discussed. @@ -35,7 +35,7 @@ First Timers You are here to help, but you are a new contributor! No worries, we always welcome newcomer contributors. We maintain some -`good first issues `_ +`good first issues `_ and encourage new contributors to work on those issues for a smooth start. .. tip:: @@ -47,15 +47,20 @@ Code Contributions For more established contributors, you can contribute to the codebase in several ways: -- Report a `bug `_; just remember to be as +- Report a `bug `_; just remember to be as specific as possible. -- Submit a `bug fix `_ for any existing +- Submit a `bug fix `_ for any existing issue. -- Create a `new issue `_ to request a +- Create a `new issue `_ to request a feature, submit a feedback, or ask a question. +* Want to add support for a new importer pipeline? See the detailed tutorial here: + :ref:`tutorial_add_importer_pipeline`. +* Interested adding a new improver pipeline? Check out the tutorial here: + :ref:`tutorial_add_improver_pipeline`. + .. note:: - Make sure to check existing `issues `_, + Make sure to check existing `issues `_, to confirm whether a given issue or a question has previously been discussed. @@ -84,582 +89,12 @@ Helpful Resources - Review our `comprehensive guide `_ for more details on how to add quality contributions to our codebase and documentation -- Check this free resource on `how to contribute to an open source project on github `_ +- Check this free resource on `How to contribute to an open source project on github `_ - Follow `this wiki page `_ on how to write good commit messages - `Pro Git book `_ - `How to write a good bug report `_ -.. _tutorial_add_a_new_importer: - -Add a new importer -------------------- - -This tutorial contains all the things one should know to quickly implement an importer. -Many internal details about importers can be found inside the -:file:`vulnerabilites/importer.py` file. -Make sure to go through :ref:`importer-overview` before you begin writing one. - -TL;DR -------- - -#. Create a new :file:`vulnerabilities/importers/{importer_name.py}` file. -#. Create a new importer subclass inheriting from the ``Importer`` superclass defined in - ``vulnerabilites.importer``. It is conventional to end an importer name with *Importer*. -#. Specify the importer license. -#. Implement the ``advisory_data`` method to process the data source you are - writing an importer for. -#. Add the newly created importer to the importers registry at - ``vulnerabilites/importers/__init__.py`` - -.. _tutorial_add_a_new_importer_prerequisites: - -Prerequisites --------------- - -Before writing an importer, it is important to familiarize yourself with the following concepts. - -PackageURL -^^^^^^^^^^^^ - -VulnerableCode extensively uses Package URLs to identify a package. See the -`PackageURL specification `_ and its `Python implementation -`_ for more details. - -**Example usage:** - -.. code:: python - - from packageurl import PackageURL - purl = PackageURL(name="ffmpeg", type="deb", version="1.2.3") - - -AdvisoryData -^^^^^^^^^^^^^ - -``AdvisoryData`` is an intermediate data format: -it is expected that your importer will convert the raw scraped data into ``AdvisoryData`` objects. -All the fields in ``AdvisoryData`` dataclass are optional; it is the importer's resposibility to -ensure that it contains meaningful information about a vulnerability. - -AffectedPackage -^^^^^^^^^^^^^^^^ - -``AffectedPackage`` data type is used to store a range of affected versions and a fixed version of a -given package. For all version-related data, `univers `_ library -is used. - -Univers -^^^^^^^^ - -`univers `_ is a Python implementation of the `vers specification `_. -It can parse and compare all the package versions and all the ranges, -from debian, npm, pypi, ruby and more. -It processes all the version range specs and expressions. - -Importer -^^^^^^^^^ - -All the generic importers need to implement the ``Importer`` class. -For ``Git`` or ``Oval`` data source, ``GitImporter`` or ``OvalImporter`` could be implemented. - -.. note:: - - ``GitImporter`` and ``OvalImporter`` need a complete rewrite. - Interested in :ref:`contributing` ? - -Writing an importer ---------------------- - -Create Importer Source File -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -All importers are located in the :file:`vulnerabilites/importers` directory. -Create a new file to put your importer code in. -Generic importers are implemented by writing a subclass for the ``Importer`` superclass and -implementing the unimplemented methods. - -Specify the Importer License -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Importers scrape data off the internet. In order to make sure the data is useable, a license -must be provided. -Populate the ``spdx_license_expression`` with the appropriate value. -The SPDX license identifiers can be found at https://spdx.org/licenses/. - -.. note:: - An SPDX license identifier by itself is a valid licence expression. In case you need more complex - expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ - -Implement the ``advisory_data`` Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``advisory_data`` method scrapes the advisories from the data source this importer is -targeted at. -It is required to return an *Iterable of AdvisoryData objects*, and thus it is a good idea to yield -from this method after creating each AdvisoryData object. - -At this point, an example importer will look like this: - -:file:`vulnerabilites/importers/example.py` - -.. code-block:: python - - from typing import Iterable - - from packageurl import PackageURL - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import Importer - - - class ExampleImporter(Importer): - - spdx_license_expression = "BSD-2-Clause" - - def advisory_data(self) -> Iterable[AdvisoryData]: - return [] - -This importer is only a valid skeleton and does not import anything at all. - -Let us implement another dummy importer that actually imports some data. - -Here we have a ``dummy_package`` which follows ``NginxVersionRange`` and ``SemverVersion`` for -version management from `univers `_. - -.. note:: - - It is possible that the versioning scheme you are targeting has not yet been - implemented in the `univers `_ library. - If this is the case, you will need to head over there and implement one. - -.. code-block:: python - - from datetime import datetime - from datetime import timezone - from typing import Iterable - - import requests - from packageurl import PackageURL - from univers.version_range import NginxVersionRange - from univers.versions import SemverVersion - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import AffectedPackage - from vulnerabilities.importer import Importer - from vulnerabilities.importer import Reference - from vulnerabilities.importer import VulnerabilitySeverity - from vulnerabilities.severity_systems import SCORING_SYSTEMS - - - class ExampleImporter(Importer): - - spdx_license_expression = "BSD-2-Clause" - - def advisory_data(self) -> Iterable[AdvisoryData]: - raw_data = fetch_advisory_data() - for data in raw_data: - yield parse_advisory_data(data) - - - def fetch_advisory_data(): - return [ - { - "id": "CVE-2021-23017", - "summary": "1-byte memory overwrite in resolver", - "advisory_severity": "medium", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", - "published_on": "14-02-2021 UTC", - }, - { - "id": "CVE-2021-1234", - "summary": "Dummy advisory", - "advisory_severity": "high", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://example.com/cve-2021-1234", - "published_on": "06-10-2021 UTC", - }, - ] - - - def parse_advisory_data(raw_data) -> AdvisoryData: - purl = PackageURL(type="example", name="dummy_package") - affected_version_range = NginxVersionRange.from_native(raw_data["vulnerable"]) - fixed_version = SemverVersion(raw_data["fixed"]) - affected_package = AffectedPackage( - package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version - ) - severity = VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], value=raw_data["advisory_severity"] - ) - references = [Reference(url=raw_data["reference"], severities=[severity])] - date_published = datetime.strptime(raw_data["published_on"], "%d-%m-%Y %Z").replace( - tzinfo=timezone.utc - ) - - return AdvisoryData( - aliases=[raw_data["id"]], - summary=raw_data["summary"], - affected_packages=[affected_package], - references=references, - date_published=date_published, - ) - - -.. note:: - - | Use ``make valid`` to format your new code using black and isort automatically. - | Use ``make check`` to check for formatting errors. - -Register the Importer -^^^^^^^^^^^^^^^^^^^^^^ - -Finally, register your importer in the importer registry at -:file:`vulnerabilites/importers/__init__.py` - -.. code-block:: python - :emphasize-lines: 1, 4 - - from vulnerabilities.importers import example - from vulnerabilities.importers import nginx - - IMPORTERS_REGISTRY = [nginx.NginxImporter, example.ExampleImporter] - - IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} - -Congratulations! You have written your first importer. - -Run Your First Importer -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If everything went well, you will see your importer in the list of available importers. - -.. code-block:: console - :emphasize-lines: 5 - - $ ./manage.py import --list - - Vulnerability data can be imported from the following importers: - vulnerabilities.importers.nginx.NginxImporter - vulnerabilities.importers.example.ExampleImporter - -Now, run the importer. - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -See :ref:`command_line_interface` for command line usage instructions. -Enable Debug Logging (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For more visibility, turn on debug logs in :file:`vulnerablecode/settings.py`. - -.. code-block:: python - - DEBUG = True - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - } - -Invoke the import command now and you will see (in a fresh database): - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Starting import for vulnerabilities.importers.example.ExampleImporter - [*] New Advisory with aliases: ['CVE-2021-23017'], created_by: vulnerabilities.importers.example.ExampleImporter - [*] New Advisory with aliases: ['CVE-2021-1234'], created_by: vulnerabilities.importers.example.ExampleImporter - Finished import for vulnerabilities.importers.example.ExampleImporter. Imported 2 advisories. - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -.. _tutorial_add_a_new_improver: - -Add a new improver ---------------------- - -This tutorial contains all the things one should know to quickly -implement an improver. -Many internal details about improvers can be found inside the -:file:`vulnerabilites/improver.py` file. -Make sure to go through :ref:`improver-overview` before you begin writing one. - -TL;DR -------- - -#. Locate the importer that this improver will be improving data of at - :file:`vulnerabilities/importers/{importer_name.py}` file. -#. Create a new improver subclass inheriting from the ``Improver`` superclass defined in - ``vulnerabilites.improver``. It is conventional to end an improver name with *Improver*. -#. Implement the ``interesting_advisories`` property to return a QuerySet of imported data - (``Advisory``) you are interested in. -#. Implement the ``get_inferences`` method to return an iterable of ``Inference`` objects for the - given ``AdvisoryData``. -#. Add the newly created improver to the improvers registry at - ``vulnerabilites/improvers/__init__.py``. - -Prerequisites --------------- - -Before writing an improver, it is important to familiarize yourself with the following concepts. - -Importer -^^^^^^^^^^ - -Importers are responsible for scraping vulnerability data from various data sources without creating -a complete relational model between vulnerabilites and their fixes and storing them in a structured -fashion. These data are stored in the ``Advisory`` model and can be converted to an equivalent -``AdvisoryData`` for various use cases. -See :ref:`importer-overview` for a brief overview on importers. - -Importer Prerequisites -^^^^^^^^^^^^^^^^^^^^^^^ - -Improvers consume data produced by importers, and thus it is important to familiarize yourself with -:ref:`Importer Prerequisites `. - -Inference -^^^^^^^^^^^ - -Inferences express the contract between the improvers and the improve runner framework. -An inference is intended to contain data points about a vulnerability without any uncertainties, -which means that one inference will target one vulnerability with the specific relevant affected and -fixed packages (in the form of `PackageURLs `_). -There is no notion of version ranges here: all package versions must be explicitly specified. - -Because this concrete relationship is rarely available anywhere upstream, we have to *infer* -these values, thus the name. -As inferring something is not always perfect, an Inference also comes with a confidence score. - -Improver -^^^^^^^^^ - -All the Improvers must inherit from ``Improver`` superclass and implement the -``interesting_advisories`` property and the ``get_inferences`` method. - -Writing an improver ---------------------- - -Locate the Source File -^^^^^^^^^^^^^^^^^^^^^^^^ - -If the improver will be working on data imported by a specific importer, it will be located in -the same file at :file:`vulnerabilites/importers/{importer-name.py}`. Otherwise, if it is a -generic improver, create a new file :file:`vulnerabilites/improvers/{improver-name.py}`. - -Explore Package Managers (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your Improver depends on the discrete versions of a package, the package managers' VersionAPI -located at :file:`vulnerabilites/package_managers.py` could come in handy. You will need to -instantiate the relevant ``VersionAPI`` in the improver's constructor and use it later in the -implemented methods. See an already implemented improver (NginxBasicImprover) for an example usage. - -Implement the ``interesting_advisories`` Property -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This property is intended to return a QuerySet of ``Advisory`` on which the ``Improver`` is -designed to work. - -For example, if the improver is designed to work on Advisories imported by ``ExampleImporter``, -the property can be implemented as - -.. code-block:: python - - class ExampleBasicImprover(Improver): - - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=ExampleImporter.qualified_name) - -Implement the ``get_inferences`` Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The framework calls ``get_inferences`` method for every ``AdvisoryData`` that is obtained from -the ``Advisory`` QuerySet returned by the ``interesting_advisories`` property. - -It is expected to return an iterable of ``Inference`` objects for the given ``AdvisoryData``. To -avoid storing a lot of Inferences in memory, it is preferable to yield from this method. - -A very simple Improver that processes all Advisories to create the minimal relationships that can -be obtained by existing data can be found at :file:`vulnerabilites/improvers/default.py`, which is -an example of a generic improver. For a more sophisticated and targeted example, you can look -at an already implemented improver (e.g., :file:`vulnerabilites/importers/nginx.py`). - -Improvers are not limited to improving discrete versions and may also improve ``aliases``. -One such example, improving the importer written in the :ref:`importer tutorial -`, is shown below. - -.. code-block:: python - - from datetime import datetime - from datetime import timezone - from typing import Iterable - - import requests - from django.db.models.query import QuerySet - from packageurl import PackageURL - from univers.version_range import NginxVersionRange - from univers.versions import SemverVersion - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.improver import MAX_CONFIDENCE - from vulnerabilities.improver import Improver - from vulnerabilities.improver import Inference - from vulnerabilities.models import Advisory - from vulnerabilities.severity_systems import SCORING_SYSTEMS - - - class ExampleImporter(Importer): - ... - - - class ExampleAliasImprover(Improver): - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=ExampleImporter.qualified_name) - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - for alias in advisory_data.aliases: - new_aliases = fetch_additional_aliases(alias) - aliases = new_aliases + [alias] - yield Inference(aliases=aliases, confidence=MAX_CONFIDENCE) - - - def fetch_additional_aliases(alias): - alias_map = { - "CVE-2021-23017": ["PYSEC-1337", "CERTIN-1337"], - "CVE-2021-1234": ["ANONSEC-1337", "CERTDES-1337"], - } - return alias_map.get(alias) - - -.. note:: - - | Use ``make valid`` to format your new code using black and isort automatically. - | Use ``make check`` to check for formatting errrors. - -Register the Improver -^^^^^^^^^^^^^^^^^^^^^^ - -Finally, register your improver in the improver registry at -:file:`vulnerabilites/improvers/__init__.py`. - -.. code-block:: python - :emphasize-lines: 7 - - from vulnerabilities import importers - from vulnerabilities.improvers import default - - IMPROVERS_REGISTRY = [ - default.DefaultImprover, - importers.nginx.NginxBasicImprover, - importers.example.ExampleAliasImprover, - ] - - IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} - -Congratulations! You have written your first improver. - -Run Your First Improver -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If everything went well, you will see your improver in the list of available improvers. - -.. code-block:: console - :emphasize-lines: 6 - - $ ./manage.py improve --list - - Vulnerability data can be processed by these available improvers: - vulnerabilities.improvers.default.DefaultImprover - vulnerabilities.importers.nginx.NginxBasicImprover - vulnerabilities.importers.example.ExampleAliasImprover - -Before running the improver, make sure you have imported the data. An improver cannot improve if -there is nothing imported. - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -Now, run the improver. - -.. code-block:: console - - $ ./manage.py improve vulnerabilities.importers.example.ExampleAliasImprover - - Improving data using vulnerabilities.importers.example.ExampleAliasImprover - Successfully improved data using vulnerabilities.importers.example.ExampleAliasImprover - -See :ref:`command_line_interface` for command line usage instructions. - -Enable Debug Logging (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For more visibility, turn on debug logs in :file:`vulnerablecode/settings.py`. - -.. code-block:: python - - DEBUG = True - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - } - -Invoke the improve command now and you will see (in a fresh database, after importing): - -.. code-block:: console - - $ ./manage.py improve vulnerabilities.importers.example.ExampleAliasImprover - - Improving data using vulnerabilities.importers.example.ExampleAliasImprover - Running improver: vulnerabilities.importers.example.ExampleAliasImprover - Improving advisory id: 1 - New alias for : PYSEC-1337 - New alias for : CVE-2021-23017 - New alias for : CERTIN-1337 - Improving advisory id: 2 - New alias for : CERTDES-1337 - New alias for : ANONSEC-1337 - New alias for : CVE-2021-1234 - Finished improving using vulnerabilities.importers.example.ExampleAliasImprover. - Successfully improved data using vulnerabilities.importers.example.ExampleAliasImprover - -.. note:: - Even though CVE-2021-23017 and CVE-2021-1234 are not supplied by this improver, the output above shows them - because we left out running the ``DefaultImprover`` in the example. The ``DefaultImprover`` - inserts minimal data found via the importers in the database (here, the above two CVEs). Run - importer, DefaultImprover and then your improver in this sequence to avoid this anomaly. diff --git a/docs/source/index.rst b/docs/source/index.rst index be51eca80..b20b1c9b5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -29,6 +29,14 @@ In this documentation you will find information on: faq misc +.. toctree:: + :maxdepth: 2 + :caption: Tutorial + + tutorial_add_importer_pipeline + tutorial_add_improver_pipeline + + .. toctree:: :maxdepth: 2 :caption: Reference Documentation diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 638276c5d..655504d3d 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -6,7 +6,7 @@ Installation .. warning:: VulnerableCode is going through a major structural change and the installations are likely to not produce enough results. - This is tracked in https://github.com/nexB/vulnerablecode/issues/597 + This is tracked in https://github.com/aboutcode-org/vulnerablecode/issues/597 Welcome to **VulnerableCode** installation guide! This guide describes how to install VulnerableCode on various platforms. @@ -40,25 +40,25 @@ Build the Image VulnerableCode is distributed with ``Dockerfile`` and ``docker-compose.yml`` files required for the creation of the Docker image. -Clone the git `VulnerableCode repo `_, +Clone the git `VulnerableCode repo `_, create an environment file, and build the Docker image:: - git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode + git clone https://github.com/aboutcode-org/vulnerablecode.git && cd vulnerablecode make envfile - docker-compose build + docker compose build .. note:: The image will need to be re-built when the VulnerableCode app source code is modified or updated via - ``docker-compose build --no-cache vulnerablecode`` + ``docker compose build --no-cache vulnerablecode`` Run the App ^^^^^^^^^^^ **Run your image** as a container:: - docker-compose up + docker compose up At this point, the VulnerableCode app should be running at port ``8000`` on your Docker host. @@ -90,7 +90,7 @@ Execute a Command You can execute a one of ``manage.py`` commands through the Docker command line interface, for example:: - docker-compose run vulnerablecode ./manage.py import --list + docker compose run vulnerablecode ./manage.py import --list .. note:: Refer to the :ref:`command_line_interface` section for the full list of commands. @@ -98,7 +98,7 @@ interface, for example:: Alternatively, you can connect to the Docker container ``bash`` and run commands from there:: - docker-compose run vulnerablecode bash + docker compose run vulnerablecode bash ./manage.py import --list @@ -146,9 +146,9 @@ Make sure those are installed:: Clone and Configure ^^^^^^^^^^^^^^^^^^^ -Clone the `VulnerableCode Git repository `_:: +Clone the `VulnerableCode Git repository `_:: - git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode + git clone https://github.com/aboutcode-org/vulnerablecode.git && cd vulnerablecode Install the required dependencies:: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index ca61bdb0f..5143343a6 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -110,4 +110,4 @@ How can I contribute to VulnerableCode? --------------------------------------- Please get in touch on our `Gitter channel `__. -You can review or get the code and report issues at our `GitHub repo `__. +You can review or get the code and report issues at our `GitHub repo `__. diff --git a/docs/source/soc_gsoc21.rst b/docs/source/soc_gsoc21.rst index 3ccd2b039..84de393de 100644 --- a/docs/source/soc_gsoc21.rst +++ b/docs/source/soc_gsoc21.rst @@ -4,7 +4,7 @@ Google Summer of Code 2021 Final Report Organization - `AboutCode `_ ----------------------------------------------------------- | `Hritik Vijay `_ -| Project: `VulnerableCode `_ +| Project: `VulnerableCode `_ Overview --------- @@ -30,7 +30,7 @@ structure:: Yielding an average of 93% reduction in time (14x faster) -More: https://github.com/nexB/vulnerablecode/pull/478 +More: https://github.com/aboutcode-org/vulnerablecode/pull/478 Speed up upstream tests ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -50,7 +50,7 @@ usage cap. In the end, this is a trade off between resource usage and data accuracy. This brings major performance improvement during the test. | Before: ~6hrs, now ~9 minutes -| More: https://github.com/nexB/vulnerablecode/pull/490 +| More: https://github.com/aboutcode-org/vulnerablecode/pull/490 Improve Docker Configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -63,8 +63,8 @@ using a docker image. The current configuration makes use of files like over any unnecessary files for deployment. | More: -| https://github.com/nexB/vulnerablecode/pull/497 -| https://github.com/nexB/vulnerablecode/pull/521 +| https://github.com/aboutcode-org/vulnerablecode/pull/497 +| https://github.com/aboutcode-org/vulnerablecode/pull/521 Add Makefile ^^^^^^^^^^^^^ @@ -78,8 +78,8 @@ relevant part of the documentation and updated settings to reject insecure deployments. | More: -| https://github.com/nexB/vulnerablecode/pull/497 -| https://github.com/nexB/vulnerablecode/pull/523 +| https://github.com/aboutcode-org/vulnerablecode/pull/497 +| https://github.com/aboutcode-org/vulnerablecode/pull/523 Use svn to collects tags in GitHubTagsAPI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -87,11 +87,11 @@ Surprisingly, GitHub allows svn requests to repositories. Now we can have all the tags with a single request. This is much more efficient and gentle to the APIs. This was as issue since the importers based on GithubDataSource were `failing -`_ because of being rate +`_ because of being rate limited by GitHub. | `Philippe `_, thank you so much for the suggestion -| More: https://github.com/nexB/vulnerablecode/pull/508 +| More: https://github.com/aboutcode-org/vulnerablecode/pull/508 Separate import and improve operations - WIP ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,14 +106,14 @@ As a bonus, writing importers will be very easy and welcome more contributors to the project. As of writing this report, this remains a work in progress which will be finished very soon. -More: https://github.com/nexB/vulnerablecode/pull/525 +More: https://github.com/aboutcode-org/vulnerablecode/pull/525 Others ^^^^^^^ -- helper: split_markdown_front_matter: https://github.com/nexB/vulnerablecode/pull/443 -- Dump yaml in favor of saneyaml https://github.com/nexB/vulnerablecode/pull/452 -- Refactor package_managers https://github.com/nexB/vulnerablecode/pull/495/commits -- Importers bugfix https://github.com/nexB/vulnerablecode/pull/505 +- helper: split_markdown_front_matter: https://github.com/aboutcode-org/vulnerablecode/pull/443 +- Dump yaml in favor of saneyaml https://github.com/aboutcode-org/vulnerablecode/pull/452 +- Refactor package_managers https://github.com/aboutcode-org/vulnerablecode/pull/495/commits +- Importers bugfix https://github.com/aboutcode-org/vulnerablecode/pull/505 Pre GSoC ---------- @@ -123,14 +123,14 @@ exploring the codebase, I realized that there is a lot of room for improvement. Thus I looked for simple improvements and bugs to fix in the early stage, which were: -- `Correct API docs path and fix pytest invocation `_ -- `Explicity provide lxml parser to beautifulsoup `_ -- `Make sure vulnerability id is_cve or is_vulcoid `_ -- `Fix istio importer `_ (cleared a huge confusion about the codebase) -- `Add me to AUTHORS `_ (Should've done this a lot earlier) -- `Add unspecified scoring system `_ -- `Fix redhat import failure `_ (This one took a *lot* of effort to pinpoint) -- `expose find_all_cve helper `_ +- `Correct API docs path and fix pytest invocation `_ +- `Explicity provide lxml parser to beautifulsoup `_ +- `Make sure vulnerability id is_cve or is_vulcoid `_ +- `Fix istio importer `_ (cleared a huge confusion about the codebase) +- `Add me to AUTHORS `_ (Should've done this a lot earlier) +- `Add unspecified scoring system `_ +- `Fix redhat import failure `_ (This one took a *lot* of effort to pinpoint) +- `expose find_all_cve helper `_ Post GSoC - Future Plans and what's left ------------------------------------------- @@ -167,7 +167,7 @@ I really enjoyed working on the project. There were ups and downs when I met some weird bugs but every one of them taught me something new about Python, Django and programming in general. The best part of working with my amazing mentors - Philippe and Shivam - were the `weekly meets -`_ +`_ where we would together try to figure out how to proceed with the development. I learned something new with every call and interaction we had. Thank you so much my mentors for providing a very smooth experience and Google for showing diff --git a/docs/source/tutorial_add_importer_pipeline.rst b/docs/source/tutorial_add_importer_pipeline.rst new file mode 100644 index 000000000..3f5b6b785 --- /dev/null +++ b/docs/source/tutorial_add_importer_pipeline.rst @@ -0,0 +1,371 @@ +.. _tutorial_add_importer_pipeline: + +Add a new pipeline to import advisories +======================================== + + +TL;DR +------- + +#. Create a new file ``{name}_importer.py`` inside **vulnerabilities/pipelines/**. +#. Create a new importer pipeline by inheriting **VulnerableCodeBaseImporterPipeline** + defined in **vulnerabilities.pipelines**. By convention the importer pipeline + class should end with **ImporterPipeline**. +#. Specify the license of upstream data being imported. +#. Implement the ``advisories_count`` and ``collect_advisories`` methods. +#. Add the newly created importer pipeline to the importers registry at + **vulnerabilities/importers/__init__.py** + + +Pipeline +-------- + +We use `aboutcode.pipeline `_ +for importing and improving data. At a very high level, a working pipeline contains classmethod +``steps`` that defines what steps to run and in what order. These steps are essentially just +functions. Pipeline provides an easy and effective way to log events inside these steps (it +automatically handles rendering and dissemination for these logs.) + +It also includes built-in progress indicator, which is essential since some of the jobs we run +in the pipeline are long-running tasks that require proper progress indicators. Pipeline provides +way to seamlessly records the progress (it automatically takes care of rendering and dissemination +of these progress). + +Additionally, the pipeline offers a consistent structure, making it easy to run these pipeline steps +with message queue like RQ and store all events related to a particular pipeline for +debugging/improvements. + +This tutorial contains all the things one should know to quickly implement an importer pipeline. +Many internal details about importer pipeline can be found inside the +`vulnerabilities/pipelines/__init__.py +`_ file. + + +.. _tutorial_add_importer_pipeline_prerequisites: + +Prerequisites +-------------- + +Before writing pipeline to import advisories, it is important to familiarize yourself with +the following concepts. + +PackageURL +~~~~~~~~~~ + +VulnerableCode extensively uses Package URLs to identify a package. See the +`PackageURL specification `_ and its `Python implementation +`_ for more details. + +**Example usage:** + +.. code:: python + + from packageurl import PackageURL + purl = PackageURL(name="ffmpeg", type="deb", version="1.2.3") + + +AdvisoryData +~~~~~~~~~~~~~ + +``AdvisoryData`` is an intermediate data format: +it is expected that your importer will convert the raw scraped data into ``AdvisoryData`` objects. +All the fields in ``AdvisoryData`` dataclass are optional; it is the importer's responsibility to +ensure that it contains meaningful information about a vulnerability. + +AffectedPackage +~~~~~~~~~~~~~~~ + +``AffectedPackage`` data type is used to store a range of affected versions and a fixed version of a +given package. For all version-related data, `univers `_ library +is used. + +Univers +~~~~~~~ + +`univers `_ is a Python implementation of the `vers specification `_. +It can parse and compare all the package versions and all the ranges, +from debian, npm, pypi, ruby and more. +It processes all the version range specs and expressions. + + +Writing an Importer Pipeline +----------------------------- + + +Create file for the new importer pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All pipelines, including the importer pipeline, are located in the +`vulnerabilities/pipelines/ +`_ directory. + +The importer pipeline is implemented by subclassing **VulnerableCodeBaseImporterPipeline** +and implementing the unimplemented methods. Since most tasks, such as inserting **AdvisoryData** +into the database and creating package-vulnerability relationships, are the same regardless of +the source of the advisory, these tasks are already taken care of in the base importer pipeline, +i.e., **VulnerableCodeBaseImporterPipeline**. You can simply focus on collecting the raw data and +parsing it to create proper **AdvisoryData** objects. + + +Specify the importer license +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The pipeline scrape data off the internet. In order to make sure the data is useable, a license +must be provided. + +Populate the ``spdx_license_expression`` with the appropriate value. The SPDX license identifiers +can be found at `ScanCode LicenseDB `_. + +.. note:: + An SPDX license identifier by itself is a valid license expression. In case you need more + complex expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ + + +Implement the ``advisories_count`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``advisories_count`` method returns the total number of advisories that will be collected by +this pipeline. + +Suppose the upstream data is a single JSON file containing a list of security advisories; +in that case, you can simply return the count of security advisories in the JSON file, +and that's it. + +.. note:: + In some cases, it could be difficult to get the exact total number of advisories that would + be collected without actually processing the advisories. In such case returning the best + estimate will also work. + + **advisories_count** is used to enable a proper progress indicator and is not used beyond that. + If it is impossible (a super rare case) to compute the total advisory count beforehand, + just return ``0``. + + +Implement the ``collect_advisories`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``collect_advisories`` method collects and parses the advisories from the data source and +yield an *AdvisoryData*. + +At this point, an example importer will look like this: + +.. code-block:: python + :caption: vulnerabilities/pipelines/example_importer.py + :linenos: + :emphasize-lines: 16-17, 20-21, 23-24 + + from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + + class ExampleImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories Example.""" + + pipeline_id = "example_importer" + + root_url = "https://example.org/path/to/advisories/" + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + importer_name = "Example Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self) -> int: + raise NotImplementedError + + def collect_advisories(self) -> Iterable[AdvisoryData]: + raise NotImplementedError + + +This pipeline is only a valid skeleton and does not import anything at all. + +Let us implement a working pipeline that actually imports some data. + +Here we have a ``dummy_package`` which follows ``NginxVersionRange`` and ``SemverVersion`` for +version management from `univers `_. + +.. note:: + + It is possible that the versioning scheme you are targeting has not yet been + implemented in the `univers `_ library. + If this is the case, you will need to head over there and implement one. + +.. code-block:: python + :caption: vulnerabilities/pipelines/example_importer.py + :linenos: + :emphasize-lines: 34-35, 37-40 + + from datetime import datetime + from datetime import timezone + from typing import Iterable + + from packageurl import PackageURL + from univers.version_range import NginxVersionRange + from univers.versions import SemverVersion + + from vulnerabilities.importer import AdvisoryData + from vulnerabilities.importer import AffectedPackage + from vulnerabilities.importer import Reference + from vulnerabilities.importer import VulnerabilitySeverity + from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + from vulnerabilities.severity_systems import SCORING_SYSTEMS + + + class ExampleImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories Example.""" + + pipeline_id = "example_importer" + + root_url = "https://example.org/path/to/advisories/" + license_url = "https://example.org/license/" + spdx_license_expression = "CC-BY-4.0" + importer_name = "Example Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self) -> int: + return len(fetch_advisory_data()) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + raw_data = fetch_advisory_data() + for data in raw_data: + yield parse_advisory_data(data) + + + def fetch_advisory_data(): + return [ + { + "id": "CVE-2021-23017", + "summary": "1-byte memory overwrite in resolver", + "advisory_severity": "medium", + "vulnerable": "0.6.18-1.20.0", + "fixed": "1.20.1", + "reference": "http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", + "published_on": "14-02-2021 UTC", + }, + { + "id": "CVE-2021-1234", + "summary": "Dummy advisory", + "advisory_severity": "high", + "vulnerable": "0.6.18-1.20.0", + "fixed": "1.20.1", + "reference": "http://example.org/cve-2021-1234", + "published_on": "06-10-2021 UTC", + }, + ] + + + def parse_advisory_data(raw_data) -> AdvisoryData: + purl = PackageURL(type="example", name="dummy_package") + affected_version_range = NginxVersionRange.from_native(raw_data["vulnerable"]) + fixed_version = SemverVersion(raw_data["fixed"]) + affected_package = AffectedPackage( + package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version + ) + severity = VulnerabilitySeverity( + system=SCORING_SYSTEMS["generic_textual"], value=raw_data["advisory_severity"] + ) + references = [Reference(url=raw_data["reference"], severities=[severity])] + date_published = datetime.strptime(raw_data["published_on"], "%d-%m-%Y %Z").replace( + tzinfo=timezone.utc + ) + advisory_url = f"https://example.org/advisory/{raw_data['id']}" + + return AdvisoryData( + aliases=[raw_data["id"]], + summary=raw_data["summary"], + affected_packages=[affected_package], + references=references, + url=advisory_url, + date_published=date_published, + ) + + +.. important:: + Steps should include ``collect_and_store_advisories`` and ``import_new_advisories`` + in the order shown above. They are defined in **VulnerableCodeBaseImporterPipeline**. + + It is the **collect_and_store_advisories** that is responsible for making calls to + **collect_advisories** and **advisories_count**, and hence **collect_advisories** and + **advisories_count** should never be directly added in steps. + + +.. attention:: + + Implement ``on_failure`` to handle cleanup in case of pipeline failure. + Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage. + +.. note:: + + | Use ``make valid`` to format your new code using black and isort automatically. + | Use ``make check`` to check for formatting errors. + +Register the Importer Pipeline +------------------------------ + +Finally, register your pipeline in the importer registry at +`vulnerabilities/importers/__init__.py +`_ + +.. code-block:: python + :caption: vulnerabilities/importers/__init__.py + :linenos: + :emphasize-lines: 1, 6 + + from vulnerabilities.pipelines import example_importer + from vulnerabilities.pipelines import nginx_importer + + IMPORTERS_REGISTRY = [ + nginx_importer.NginxImporterPipeline, + example_importer.ExampleImporterPipeline, + ] + + IMPORTERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodeBaseImporterPipeline) else x.qualified_name: x + for x in IMPORTERS_REGISTRY + } + +Congratulations! You have written your first importer pipeline. + +Run Your First Importer Pipeline +-------------------------------- + +If everything went well, you will see your pipeline in the list of available importers. + +.. code-block:: console + :emphasize-lines: 5 + + $ ./manage.py import --list + + Vulnerability data can be imported from the following importers: + nginx_importer + example_importer + +Now, run the importer. + +.. code-block:: console + + $ ./manage.py import example_importer + + Importing data using example_importer + INFO 2024-10-16 10:15:10.483 Pipeline [ExampleImporterPipeline] starting + INFO 2024-10-16 10:15:10.483 Step [collect_and_store_advisories] starting + INFO 2024-10-16 10:15:10.483 Collecting 2 advisories + INFO 2024-10-16 10:15:10.498 Successfully collected 2 advisories + INFO 2024-10-16 10:15:10.498 Step [collect_and_store_advisories] completed in 0 seconds + INFO 2024-10-16 10:15:10.498 Step [import_new_advisories] starting + INFO 2024-10-16 10:15:10.499 Importing 2 new advisories + INFO 2024-10-16 10:15:10.562 Successfully imported 2 new advisories + INFO 2024-10-16 10:15:10.563 Step [import_new_advisories] completed in 0 seconds + INFO 2024-10-16 10:15:10.563 Pipeline completed in 0 seconds + + +See :ref:`command_line_interface` for command line usage instructions. diff --git a/docs/source/tutorial_add_improver_pipeline.rst b/docs/source/tutorial_add_improver_pipeline.rst new file mode 100644 index 000000000..feed3556b --- /dev/null +++ b/docs/source/tutorial_add_improver_pipeline.rst @@ -0,0 +1,275 @@ +.. _tutorial_add_improver_pipeline: + +Add pipeline to improve/enhance data +===================================== + +TL;DR +------- + +#. Create a new file ``{improver_name}.py`` inside **vulnerabilities/pipelines/**. +#. Create a new improver pipeline by inheriting **VulnerableCodePipeline** defined + in **vulnerabilities.pipelines**. +#. Implement ``steps`` **classmethod** to define what function to run and in which order. +#. Implement the individual function defined in ``steps`` +#. Add the newly created pipeline to the improvers registry at + **vulnerabilities/improvers/__init__.py**. + +Pipeline +-------- + +We use `aboutcode.pipeline `_ +for importing and improving data. At a very high level, a working pipeline contains classmethod +``steps`` that defines what steps to run and in what order. These steps are essentially just +functions. Pipeline provides an easy and effective way to log events inside these steps (it +automatically handles rendering and dissemination for these logs.) + +It also includes built-in progress indicator, which is essential since some of the jobs we run +in the pipeline are long-running tasks that require proper progress indicators. Pipeline provides +way to seamlessly records the progress (it automatically takes care of rendering and dissemination +of these progress). + +Additionally, the pipeline offers a consistent structure, making it easy to run these pipeline steps +with message queue like RQ and store all events related to a particular pipeline for +debugging/improvements. + +This tutorial contains all the things one should know to quickly implement an improver pipeline. + + +Prerequisites +------------- + +The new improver design lets you do all sorts of cool improvements and enhancements. +Some of those are: + +* Let's suppose you have a certain number of packages and vulnerabilities in your database, + and you want to make sure that the packages being shown in VulnerableCode do indeed exist + upstream. Oftentimes, we come across advisory data that contains made-up package versions. + We can write (well, we already have) a pipeline that iterates through all the packages in + VulnerableCode and labels them as ghost packages if they don't exist upstream. + + +- A basic security advisory only contains CVE/aliases, summary, fixed/affected version, and + severity. But now we can use the new pipeline to enhance the vulnerability info with exploits from + various sources like ExploitDB, Metasploit, etc. + + +* Likewise, we can have more pipelines to flag malicious/yanked packages. + + +So you see, the new improver pipeline is very powerful in what you can achieve, but as always, with +great power comes great responsibility. By design, the new improver are unconstrained, and you must +be absolutely sure of what you're doing and should have robust tests for these pipelines in place. + + +Writing an Improver Pipeline +----------------------------- + +**Scenario:** Suppose we come around a source that curates and stores the list of packages that +don't exist upstream and makes it available through the REST API endpoint +https://example.org/api/non-existent-packages, which gives a JSON response with a list of +non-existent packages. + +Let's write a pipeline that will use this source to flag these non-existent package as +ghost package. + + +Create file for the new improver pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All pipelines, including the improver pipeline, are located in the +`vulnerabilities/pipelines/ +`_ directory. + +The improver pipeline is implemented by subclassing `VulnerableCodePipeline`. + +Specify the importer license +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the improver pipeline scrapes data off the internet, we need to track the license for +the scraped data to make sure that we can legally use it. + +Populate the ``spdx_license_expression`` with the appropriate value. The SPDX license identifiers +can be found at `ScanCode LicenseDB `_. + +.. note:: + An SPDX license identifier by itself is a valid license expression. In case you need more + complex expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ + + +Add skeleton for new pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this scenario pipeline needs to do two thing fetch raw data and use that to flag those packages. + +At this point improver will look like this: + +.. code-block:: python + :caption: vulnerabilities/pipelines/flag_ghost_package_with_example_org.py + :linenos: + :emphasize-lines: 14-15, 18-19, 21-22 + + from vulnerabilities.pipelines import VulnerableCodePipeline + + class FlagGhostPackagesWithExampleOrg(VulnerableCodePipeline): + """Example improver pipeline to flag ghost packages.""" + + pipeline_id = "flag_ghost_package_with_example_org" + + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_response, + cls.flag_ghost_packages, + ) + + def fetch_response(self): + raise NotImplementedError + + def flag_ghost_packages(self): + raise NotImplementedError + + +Implement the steps +~~~~~~~~~~~~~~~~~~~ + +We will evolve our high level design by implementing ``fetch_response`` and ``flag_ghost_packages`` +methods. + +.. code-block:: python + :caption: vulnerabilities/pipelines/flag_ghost_package_with_example_org.py + :linenos: + :emphasize-lines: 20-32, 34-42 + + from vulnerabilities.models import Package + from vulnerabilities.pipelines import VulnerableCodePipeline + + + class FlagGhostPackagesWithExampleOrg(VulnerableCodePipeline): + """Example improver pipeline to flag ghost packages.""" + + pipeline_id = "flag_ghost_package_with_example_org" + + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_response, + cls.flag_ghost_packages, + ) + + def fetch_response(self): + # Since this is imaginary source we will mock the response + # In actual implementation you need to use request library to get data. + mock_response = { + "non-existent": [ + "pkg:npm/626@1.1.1", + "pkg:npm/bootstrap-tagsinput@0.8.0", + "pkg:npm/dojo@1.0.0", + "pkg:npm/dojo@1.1.0", + "pkg:npm/electron@1.8.0", + ] + } + self.fetched_data = mock_response + + def flag_ghost_packages(self): + non_existent_packages = self.fetched_data.get("non-existent", []) + + ghost_packages = Package.objects.filter(package_url__in=non_existent_packages) + ghost_package_count = ghost_packages.count() + + ghost_packages.update(is_ghost=True) + + self.log(f"Successfully flagged {ghost_package_count:,d} ghost Packages") + + +.. attention:: + + Implement ``on_failure`` to handle cleanup in case of pipeline failure. + Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage. + +.. note:: + + | Use ``make valid`` to format your new code using black and isort automatically. + | Use ``make check`` to check for formatting errors. + + +Register the Improver Pipeline +------------------------------ + +Finally, register your improver in the improver registry at +`vulnerabilities/improvers/__init__.py +`_ + + +.. code-block:: python + :caption: vulnerabilities/improvers/__init__.py + :linenos: + :emphasize-lines: 2, 6 + + from vulnerabilities.pipeline import enhance_with_kev + from vulnerabilities.pipeline import flag_ghost_package_with_example_org + + IMPROVERS_REGISTRY = [ + enhance_with_kev.VulnerabilityKevPipeline, + flag_ghost_package_with_example_org.FlagGhostPackagesWithExampleOrg, + ] + + IMPROVERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodePipeline) else x.qualified_name: x + for x in IMPROVERS_REGISTRY + } + + +Congratulations! You have written your first improver pipeline. + +Run Your First Improver Pipeline +-------------------------------- + +If everything went well, you will see your improver in the list of available improvers. + +.. code-block:: console + :emphasize-lines: 5 + + $ ./manage.py improve --list + + Vulnerability data can be processed by these available improvers: + enhance_with_kev + flag_ghost_package_with_example_org + +Now, run the improver. + +.. code-block:: console + + $ ./manage.py improve flag_ghost_package_with_example_org + + Improving data using flag_ghost_package_with_example_org + INFO 2024-10-17 14:37:54.482 Pipeline [FlagGhostPackagesWithExampleOrg] starting + INFO 2024-10-17 14:37:54.482 Step [fetch_response] starting + INFO 2024-10-17 14:37:54.482 Step [fetch_response] completed in 0 seconds + INFO 2024-10-17 14:37:54.482 Step [flag_ghost_packages] starting + INFO 2024-10-17 14:37:54.488 Successfully flagged 5 ghost Packages + INFO 2024-10-17 14:37:54.488 Step [flag_ghost_packages] completed in 0 seconds + INFO 2024-10-17 14:37:54.488 Pipeline completed in 0 seconds + + +See :ref:`command_line_interface` for command line usage instructions. + +.. tip:: + + If you need to improve package vulnerability relations created using a certain pipeline, + simply use the **pipeline_id** to filter out only those items. For example, if you want + to improve only those **AffectedByPackageRelatedVulnerability** entries that were created + by npm_importer pipeline, you can do so with the following query: + + .. code-block:: python + + AffectedByPackageRelatedVulnerability.objects.filter(created_by=NpmImporterPipeline.pipeline_id) + +.. note:: + + Make sure to use properly optimized query sets, and wherever needed, use paginated query sets. diff --git a/pyproject-aboutcode.hashid.toml b/pyproject-aboutcode.hashid.toml new file mode 100644 index 000000000..a2c304323 --- /dev/null +++ b/pyproject-aboutcode.hashid.toml @@ -0,0 +1,74 @@ +[build-system] +requires = [ "flot>=0.7.0" ] +build-backend = "flot.buildapi" + +[project] +name = "aboutcode.hashid" +version = "0.2.0" +description = "A library for aboutcode hash-based identifiers for VCID, and PURLs" +readme = "aboutcode/hashid/README.rst" +license = { text = "Apache-2.0 AND Python-2.0" } +requires-python = ">=3.8" + +authors = [ + { name = "AboutCode, nexB Inc. and others", email = "info@aboutcode.org" }, +] + +keywords = [ + "purl", + "Package-URL", + "open source", + "package", + "sca", + "scan", + "VCID", + "hash", +] + +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development", + "Topic :: Utilities", +] + +dependencies = [ + "packageurl_python >= 0.15.6", +] + +urls = { Homepage = "https://github.com/aboutcode-org/vulnerablecode" } + + +[tool.bumpversion] +current_version = "0.1.0" +allow_dirty = true + +files = [ + { filename = "pyproject-aboutcode.hashid.toml" }, +] + +[tool.flot] +includes = [ + "aboutcode/**/*", +] + +excludes = [ + # Python compiled files + "**/*.py[cod]", + "**/*.egg-info", + # Various junk and temp files + "**/.DS_Store", + "**/*~", + "**/.*.sw[po]", + "**/.ve", + "**/*.bak", + "**/.ipynb_checkpoints", + "aboutcode/hashid/python.LICENSE", + "aboutcode/hashid/tests/**/*", +] + +metadata_files = ["apache-2.0.LICENSE", "NOTICE", "aboutcode/hashid/python.LICENSE"] +editable_paths = ["aboutcode"] + diff --git a/requirements.txt b/requirements.txt index f73700e83..84ea22538 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ +aboutcode.pipeline==0.1.0 aiosignal==1.2.0 alabaster==0.7.12 -asgiref==3.5.2 +asgiref==3.8.1 asttokens==2.0.5 async-timeout==4.0.2 attrs==21.4.0 @@ -10,50 +11,59 @@ bcrypt==3.2.0 beautifulsoup4==4.10.0 binaryornot==0.4.4 black==22.3.0 -boolean.py==3.8 +bleach==6.1.0 +boolean.py==4.0 certifi==2024.7.4 cffi==1.15.0 chardet==4.0.0 charset-normalizer==2.0.12 click==8.1.2 -cryptography==42.0.4 +coreapi==2.3.3 +coreschema==0.0.4 +cryptography==43.0.1 +crispy-bootstrap4==2024.1 +cwe2==3.0.0 +dateparser==1.1.1 decorator==5.1.1 defusedxml==0.7.1 distro==1.7.0 -Django==4.1.13 -django-crispy-forms==1.10.0 -django-environ==0.8.1 -django-filter==21.1 -django-widget-tweaks==1.4.12 -djangorestframework==3.13.1 +Django==4.2.16 +django-crispy-forms==2.3 +django-environ==0.11.2 +django-filter==24.3 +django-widget-tweaks==1.5.0 +djangorestframework==3.15.2 doc8==0.11.1 -docker==5.0.3 -dockerpty==0.4.1 docopt==0.6.2 docutils==0.17.1 +drf-spectacular==0.24.2 +drf-spectacular-sidecar==2022.10.1 executing==0.8.3 +fetchcode==0.6.0 freezegun==1.2.1 frozenlist==1.3.0 gitdb==4.0.9 GitPython==3.1.41 -gunicorn==22.0.0 +gunicorn==23.0.0 idna==3.3 imagesize==1.3.0 importlib-metadata==4.11.3 iniconfig==1.1.1 ipython==8.10.0 isort==5.10.1 +itypes==1.2.0 jedi==0.18.1 Jinja2==3.1.4 jsonschema==3.2.0 -license-expression==21.6.14 +license-expression==30.3.1 lxml==4.9.1 Markdown==3.3.4 +markdown-it-py==3.0.0 MarkupSafe==2.1.1 matplotlib-inline==0.1.3 multidict==6.0.2 mypy-extensions==0.4.3 -packageurl-python==0.10.5rc1 +packageurl-python==0.15.6 packaging==21.3 paramiko==3.4.0 parso==0.8.3 @@ -90,14 +100,15 @@ smmap==5.0.0 snowballstemmer==2.2.0 soupsieve==2.3.2 Sphinx==4.5.0 -sphinx-rtd-theme==1.0.0 sphinxcontrib-applehelp==1.0.2 +sphinx-autobuild==2024.10.3 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-django2==1.5 sphinxcontrib-htmlhelp==2.0.0 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 +sphinx-rtd-theme==1.0.0 sqlparse==0.5.0 stack-data==0.2.0 stevedore==3.5.0 @@ -112,11 +123,3 @@ wcwidth==0.2.5 websocket-client==0.59.0 yarl==1.7.2 zipp==3.19.1 -dateparser==1.1.1 -fetchcode==0.3.0 -cwe2==2.0.0 -drf-spectacular-sidecar==2022.10.1 -drf-spectacular==0.24.2 -coreapi==2.3.3 -coreschema==0.0.4 -itypes==1.2.0 diff --git a/setup.cfg b/setup.cfg index a51fb5b73..a3db96abd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 33.6.3 +version = 35.1.0 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 @@ -48,31 +48,31 @@ license_files = README.rst [options] -python_requires = >=3.8 +python_requires = >=3.9 packages=find: include_package_data = true zip_safe = false install_requires = - Django>=4.0.0 + Django>=4.2.0,<=5.0 psycopg2-binary>=2.8.6 - djangorestframework>=3.12.4 + djangorestframework>=3.15.0 django-extensions>=3.2.3 - django-filter>=2.4.0 - django-widget-tweaks>=1.4.8 - django-crispy-forms>=1.10.0 - django-environ>=0.8.0 - gunicorn>=20.1.0 + django-filter>=24.0 + django-widget-tweaks>=1.5.0 + django-crispy-forms>=2.3 + crispy-bootstrap4>=2024.1 + django-environ>=0.11.0 + gunicorn>=23.0.0 # for the API doc drf-spectacular[sidecar]>=0.24.2 - coreapi>=2.3.3 #essentials - packageurl-python>=0.10.5rc1 + packageurl-python>=0.15 univers>=30.12.0 - license-expression>=21.6.14 + license-expression>=30.0.0 # file and data formats binaryornot>=0.4.4 @@ -85,12 +85,15 @@ install_requires = Markdown>=3.3.0 dateparser>=1.1.1 cvss>=2.4 - cwe2>=2.0.0 + cwe2>=3.0.0 # networking GitPython>=3.1.17 requests>=2.25.1 - fetchcode>=0.3.0 + fetchcode>=0.6.0 + + #pipeline + aboutcode.pipeline>=0.1.0 #vulntotal python-dotenv @@ -108,6 +111,7 @@ dev = Sphinx>=4.5.0 sphinx_rtd_theme>=1.0.0 sphinxcontrib-django2>=1.5 + sphinx-autobuild>=2024.10.3 # Tests pytest>=7.0.1 pytest-django>=4.5.2 @@ -119,6 +123,8 @@ dev = # debug django-debug-toolbar pyinstrument + flot + twine [options.entry_points] console_scripts = diff --git a/vulnerabilities/__init__.py b/vulnerabilities/__init__.py index bdac1cd30..20854f2ad 100644 --- a/vulnerabilities/__init__.py +++ b/vulnerabilities/__init__.py @@ -3,6 +3,6 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/admin.py b/vulnerabilities/admin.py index 19e006ae5..eecef0276 100644 --- a/vulnerabilities/admin.py +++ b/vulnerabilities/admin.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,7 +13,6 @@ from vulnerabilities.models import ApiUser from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilitySeverity @@ -35,12 +34,6 @@ class PackageAdmin(admin.ModelAdmin): search_fields = ["name"] -@admin.register(PackageRelatedVulnerability) -class PackageRelatedVulnerabilityAdmin(admin.ModelAdmin): - list_filter = ("package__type", "package__namespace") - search_fields = ["vulnerability__vulnerability_id", "package__name"] - - @admin.register(VulnerabilitySeverity) class VulnerabilitySeverityAdmin(admin.ModelAdmin): pass diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 3902e9190..1fd480ce9 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -3,16 +3,18 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from urllib.parse import unquote +from cvss.exceptions import CVSS2MalformedError +from cvss.exceptions import CVSS3MalformedError +from cvss.exceptions import CVSS4MalformedError from django.db.models import Prefetch from django_filters import rest_framework as filters from drf_spectacular.utils import extend_schema -from drf_spectacular.utils import inline_serializer from packageurl import PackageURL from packageurl import normalize_qualifiers from rest_framework import serializers @@ -20,19 +22,20 @@ from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle -from rest_framework.throttling import UserRateThrottle from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness from vulnerabilities.models import get_purl_query_lookups +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.throttling import StaffUserRateThrottle +from vulnerabilities.utils import get_severity_range class VulnerabilitySeveritySerializer(serializers.ModelSerializer): @@ -49,13 +52,25 @@ def to_representation(self, instance): class VulnerabilityReferenceSerializer(serializers.ModelSerializer): - scores = VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set") + scores = serializers.SerializerMethodField() reference_url = serializers.CharField(source="url") class Meta: model = VulnerabilityReference fields = ["reference_url", "reference_id", "reference_type", "scores", "url"] + def get_scores(self, instance): + severities_related_to_reference = [ + severity + for severity in self.context.get("severities", []) + if severity.url == instance.url + ] + + return VulnerabilitySeveritySerializer( + severities_related_to_reference, + many=True, + ).data + class BaseResourceSerializer(serializers.HyperlinkedModelSerializer): """ @@ -82,29 +97,23 @@ def get_resource_url(self, instance): return resource_url -class MinimalPackageSerializer(BaseResourceSerializer): +class VulnVulnIDSerializer(serializers.Serializer): """ - Used for nesting inside vulnerability focused APIs. + Serializer for the series of vulnerability IDs. """ - def get_affected_vulnerabilities(self, package): - parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities") or [] + vulnerability = serializers.CharField(source="vulnerability_id") - affected_vulnerabilities = [ - self.get_vulnerability(vuln) for vuln in parent_affected_vulnerabilities - ] - - return affected_vulnerabilities + class Meta: + fields = ["vulnerability"] - def get_vulnerability(self, vuln): - affected_vulnerability = {} - vulnerability = vuln.get("vulnerability") - if vulnerability: - affected_vulnerability["vulnerability"] = vulnerability.vulnerability_id - return affected_vulnerability +class MinimalPackageSerializer(BaseResourceSerializer): + """ + Used for nesting inside vulnerability focused APIs. + """ - affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") + affected_by_vulnerabilities = VulnVulnIDSerializer(source="affecting_vulns", many=True) purl = serializers.CharField(source="package_url") @@ -140,22 +149,43 @@ class VulnSerializerRefsAndSummary(BaseResourceSerializer): Lookup vulnerabilities references by aliases (such as a CVE). """ - def to_representation(self, instance): - data = super().to_representation(instance) - aliases = [alias["alias"] for alias in data["aliases"]] - data["aliases"] = aliases - return data - fixed_packages = MinimalPackageSerializer( many=True, source="filtered_fixed_packages", read_only=True ) - references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") - aliases = AliasSerializer(many=True, source="alias") + references = serializers.SerializerMethodField() + + aliases = serializers.SerializerMethodField() + + def get_aliases(self, obj): + # Assuming `obj.aliases` is a queryset of `Alias` objects + return [alias.alias for alias in obj.aliases.all()] + + def get_references(self, vulnerability): + references = vulnerability.vulnerabilityreference_set.all() + severities = vulnerability.severities.all() + + serialized_references = VulnerabilityReferenceSerializer( + references, + context={"severities": severities}, + many=True, + ).data + + return serialized_references class Meta: model = Vulnerability - fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + fields = [ + "url", + "vulnerability_id", + "summary", + "references", + "fixed_packages", + "aliases", + "risk_score", + "exploitability", + "weighted_severity", + ] class WeaknessSerializer(serializers.HyperlinkedModelSerializer): @@ -177,10 +207,23 @@ def to_representation(self, instance): return representation -class KEVSerializer(serializers.ModelSerializer): +class ExploitSerializer(serializers.ModelSerializer): class Meta: - model = Kev - fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"] + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] class VulnerabilitySerializer(BaseResourceSerializer): @@ -188,11 +231,11 @@ class VulnerabilitySerializer(BaseResourceSerializer): many=True, source="filtered_fixed_packages", read_only=True ) affected_packages = MinimalPackageSerializer(many=True, read_only=True) - - references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") + references = serializers.SerializerMethodField() aliases = AliasSerializer(many=True, source="alias") - kev = KEVSerializer(read_only=True) + exploits = ExploitSerializer(many=True, read_only=True) weaknesses = WeaknessSerializer(many=True) + severity_range_score = serializers.SerializerMethodField() def to_representation(self, instance): data = super().to_representation(instance) @@ -200,12 +243,44 @@ def to_representation(self, instance): weaknesses = data.get("weaknesses", []) data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None] - kev = data.get("kev", None) - if not kev: - data.pop("kev") - return data + def get_references(self, vulnerability): + references = vulnerability.vulnerabilityreference_set.all() + severities = vulnerability.severities.all() + + serialized_references = VulnerabilityReferenceSerializer( + references, + context={"severities": severities}, + many=True, + ).data + + return serialized_references + + def get_severity_range_score(self, instance): + severity_vectors = [] + severity_values = set() + for s in instance.severities.all(): + if s.scoring_system == EPSS.identifier: + continue + + if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: + try: + vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ): + pass + + if s.value: + severity_values.add(s.value) + severity_range = get_severity_range(severity_values) + return severity_range + class Meta: model = Vulnerability fields = [ @@ -217,7 +292,11 @@ class Meta: "affected_packages", "references", "weaknesses", - "kev", + "exploits", + "severity_range_score", + "exploitability", + "weighted_severity", + "risk_score", ] @@ -226,25 +305,8 @@ class PackageSerializer(BaseResourceSerializer): Lookup software package using Package URLs """ - def to_representation(self, instance): - data = super().to_representation(instance) - data["qualifiers"] = normalize_qualifiers(data["qualifiers"], encode=False) - - return data - - next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable") - - def get_next_non_vulnerable(self, package): - next_non_vulnerable = package.fixed_package_details.get("next_non_vulnerable", None) - if next_non_vulnerable: - return next_non_vulnerable.version - - latest_non_vulnerable_version = serializers.SerializerMethodField("get_latest_non_vulnerable") - - def get_latest_non_vulnerable(self, package): - latest_non_vulnerable = package.fixed_package_details.get("latest_non_vulnerable", None) - if latest_non_vulnerable: - return latest_non_vulnerable.version + next_non_vulnerable_version = serializers.CharField(read_only=True) + latest_non_vulnerable_version = serializers.CharField(read_only=True) purl = serializers.CharField(source="package_url") @@ -252,8 +314,13 @@ def get_latest_non_vulnerable(self, package): fixing_vulnerabilities = serializers.SerializerMethodField("get_fixing_vulnerabilities") + qualifiers = serializers.SerializerMethodField() + is_vulnerable = serializers.BooleanField() + def get_qualifiers(self, package): + return normalize_qualifiers(package.qualifiers, encode=False) + def get_fixed_packages(self, package): """ Return a queryset of all packages that fix a vulnerability with @@ -266,7 +333,7 @@ def get_fixed_packages(self, package): type=package.type, qualifiers=package.qualifiers, subpath=package.subpath, - packagerelatedvulnerability__fix=True, + fixingpackagerelatedvulnerability__isnull=False, ) .with_is_vulnerable() .distinct() @@ -279,10 +346,13 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: otherwise return vulnerabilities fixed by the `package`. """ fixed_packages = self.get_fixed_packages(package=package) - qs = package.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + if not fix: + qs = package.affected_by_vulnerabilities.all() + else: + qs = package.fixing_vulnerabilities.all() qs = qs.prefetch_related( Prefetch( - "packages", + "fixed_by_packages", queryset=fixed_packages, to_attr="filtered_fixed_packages", ) @@ -297,6 +367,10 @@ def get_fixing_vulnerabilities(self, package) -> dict: """ Return a mapping of vulnerabilities fixed in the given `package`. """ + # Ghost package should not fix any vulnerability. + if package.is_ghost: + return [] + return self.get_vulnerabilities_for_a_package(package=package, fix=True) def get_affected_vulnerabilities(self, package) -> dict: @@ -335,10 +409,9 @@ class Meta: "latest_non_vulnerable_version", "affected_by_vulnerabilities", "fixing_vulnerabilities", + "risk_score", ] - is_vulnerable = serializers.BooleanField() - class PackageFilterSet(filters.FilterSet): purl = filters.CharFilter(method="filter_purl") @@ -353,7 +426,6 @@ class Meta: "qualifiers", "subpath", "purl", - "packagerelatedvulnerability__fix", ] def filter_purl(self, queryset, name, value): @@ -476,12 +548,15 @@ def bulk_search(self, request): @action(detail=False, methods=["get"]) def all(self, request): """ - Return the Package URLs of all packages known to be vulnerable. + Return a list of Package URLs of vulnerable packages. """ - vulnerable_packages = ( - Package.objects.vulnerable().only("package_url").distinct().with_is_vulnerable() + vulnerable_purls = ( + Package.objects.vulnerable() + .only("package_url") + .order_by("package_url") + .distinct() + .values_list("package_url", flat=True) ) - vulnerable_purls = [str(package.package_url) for package in vulnerable_packages] return Response(vulnerable_purls) @extend_schema( @@ -568,7 +643,14 @@ def get_fixed_packages_qs(self): Filter the packages that fixes a vulnerability on fields like name, namespace and type. """ - return self.get_packages_qs().filter(packagerelatedvulnerability__fix=True) + return ( + self.get_packages_qs() + .filter( + fixingpackagerelatedvulnerability__isnull=False, + is_ghost=False, + ) + .with_is_vulnerable() + ) def get_packages_qs(self): """ @@ -591,13 +673,12 @@ def get_queryset(self): super() .get_queryset() .prefetch_related( - Prefetch( - "packages", - queryset=self.get_packages_qs(), - ), "weaknesses", + "references", + "exploits", + "severities", Prefetch( - "packages", + "fixed_by_packages", queryset=self.get_fixed_packages_qs(), to_attr="filtered_fixed_packages", ), @@ -618,17 +699,13 @@ def filter_cpe(self, queryset, name, value): return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct() -class CPEViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe) - """ +class CPEViewSet(VulnerabilityViewSet): + """Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe)""" queryset = Vulnerability.objects.filter( vulnerabilityreference__reference_id__startswith="cpe" ).distinct() - serializer_class = VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) - throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] + filterset_class = CPEFilterSet @action(detail=False, methods=["post"]) @@ -663,14 +740,10 @@ def filter_alias(self, queryset, name, value): return self.queryset.filter(aliases__alias__icontains=alias) -class AliasViewSet(viewsets.ReadOnlyModelViewSet): +class AliasViewSet(VulnerabilityViewSet): """ Lookup for vulnerabilities by vulnerability aliases such as a CVE (https://nvd.nist.gov/general/cve-process). """ - queryset = Vulnerability.objects.all() - serializer_class = VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) filterset_class = AliasFilterSet - throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py new file mode 100644 index 000000000..7a13baf42 --- /dev/null +++ b/vulnerabilities/api_extension.py @@ -0,0 +1,423 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from urllib.parse import unquote + +from django_filters import rest_framework as filters +from drf_spectacular.utils import extend_schema +from packageurl import PackageURL +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.fields import CharField +from rest_framework.fields import ListField +from rest_framework.fields import SerializerMethodField +from rest_framework.response import Response +from rest_framework.serializers import HyperlinkedModelSerializer +from rest_framework.serializers import ModelSerializer +from rest_framework.serializers import Serializer +from rest_framework.serializers import ValidationError +from rest_framework.throttling import AnonRateThrottle + +from vulnerabilities.api import BaseResourceSerializer +from vulnerabilities.models import Exploit +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness +from vulnerabilities.models import get_purl_query_lookups +from vulnerabilities.throttling import StaffUserRateThrottle + + +class SerializerExcludeFieldsMixin: + """ + A Serializer mixin that takes an additional `exclude_fields` argument to + exclude specific fields from the serialized content. + + Useful for complex serialization where a subclass just needs one less field, like a URL field. + Inspired by https://www.django-rest-framework.org/api-guide/serializers/#example + """ + + def __init__(self, *args, **kwargs): + exclude_fields = kwargs.pop("exclude_fields", []) + + super().__init__(*args, **kwargs) + + for field_name in exclude_fields: + self.fields.pop(field_name) + + +class ExcludeFieldsSerializerMixin(Serializer): + """ + A base Serializer with an `exclude_fields` attribute to + exclude specific fields from the serialized content. + + Useful for complex serialization where a subclass just needs one less field, like a URL field. + Inspired by https://www.django-rest-framework.org/api-guide/serializers/#example + """ + + exclude_fields = [] + + def handle_field(self, obj, field): + """ + Exlude fields from serialization using the ``exclude_fields`` attribute. + """ + if field.name in self.exclude_fields: + return + super().handle_field(obj, field) + + +class V2VulnerabilityReferenceSerializer(ModelSerializer): + reference_url = CharField(source="url") + + class Meta: + model = VulnerabilityReference + fields = ("reference_url", "reference_id", "reference_type") + + +class V2VulnerabilitySeveritySerializer(ModelSerializer): + score = CharField(source="value") + + class Meta: + model = VulnerabilitySeverity + fields = ("url", "score", "scoring_system", "scoring_elements", "published_at") + + +class V2WeaknessSerializer(ModelSerializer): + class Meta: + model = Weakness + fields = ("cwe",) + + +class V2WeaknessFullSerializer(ModelSerializer): + class Meta: + model = Weakness + fields = ("cwe", "name", "description") + + +class V2ExploitSerializer(ModelSerializer): + class Meta: + model = Exploit + fields = [ + "date_added", + "description", + "required_action", + "due_date", + "notes", + "known_ransomware_campaign_use", + "source_date_published", + "exploit_type", + "platform", + "source_date_updated", + "data_source", + "source_url", + ] + + +class V2VulnerabilitySerializer(ModelSerializer): + """Vulnerabilities with inlined related objects, but no package.""" + + aliases = SerializerMethodField("get_aliases") + weaknesses = V2WeaknessSerializer(many=True, source="weaknesses_set") + references = V2VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") + exploits = V2ExploitSerializer(many=True, source="weaknesses") + severities = V2VulnerabilitySeveritySerializer(many=True) + + def get_aliases(self, vulnerability): + return vulnerability.aliases.only("alias").values_list("alias", flat=True) + + def get_cwes(self, vulnerability): + return [ + w.cwe for w in vulnerability.weaknesses.only("cwe_id").values_list("cwe_id", flat=True) + ] + + class Meta: + model = Vulnerability + fields = ( + "vulnerability_id", + "aliases", + "status", + "weaknesses", + "summary", + "exploits", + "references", + "severities", + ) + + +class V2LinkedVulnerabilitySerializer(V2VulnerabilitySerializer, HyperlinkedModelSerializer): + """Vulnerabilities with a URL.""" + + class Meta: + model = Vulnerability + fields = ("url",) + V2VulnerabilitySerializer.Meta.fields + + +class V2PackageSerializer(BaseResourceSerializer): + """Package with inlined related vulnerability ids, but no other nested data.""" + + purl = CharField(source="package_url") + next_non_vulnerable_version = SerializerMethodField("get_next_non_vuln_version") + latest_non_vulnerable_version = SerializerMethodField("get_latest_non_vuln_version") + affected_by_vulnerabilities = SerializerMethodField("get_affected_by_vulns") + fixing_vulnerabilities = SerializerMethodField("get_fixing_vulns") + + class Meta: + model = Package + fields = ( + "purl", + "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", + ) + + def get_next_non_vuln_version(self, package): + if next_non_vulnerable := package.fixed_package_details.get("next_non_vulnerable"): + return next_non_vulnerable.version + + def get_latest_non_vuln_version(self, package): + if latest_non_vulnerable := package.fixed_package_details.get("latest_non_vulnerable"): + return latest_non_vulnerable.version + + def get_fixing_vulns(self, package) -> dict: + return package.fixing_vulnerabilities.only("vulnerability_id").values_list( + "vulnerability_id" + ) + + def get_affected_by_vulns(self, package) -> dict: + return package.affected_by_vulnerabilities.only("vulnerability_id").values_list( + "vulnerability_id" + ) + + +class V2LinkedPackageSerializer(V2PackageSerializer, HyperlinkedModelSerializer): + """Serialize package with a URL.""" + + class Meta: + model = Package + fields = ("url",) + V2PackageSerializer.Meta.fields + + +class V2PackageurlListSerializer(Serializer): + """List of purls.""" + + purls = ListField(child=CharField(), allow_empty=False, help_text="List of PackageURLs.") + + +class V2LookupRequestSerializer(Serializer): + """Single purl.""" + + purl = CharField(required=True, help_text="PackageURL string.") + + +class V2PackageFilterSet(filters.FilterSet): + purl = filters.CharFilter(method="filter_purl") + + class Meta: + model = Package + fields = [ + "type", + "namespace", + "name", + "version", + "qualifiers", + "subpath", + "purl", + ] + + def filter_purl(self, queryset, name, value): + purl = unquote(value) + try: + purl = PackageURL.from_string(purl) + + except ValueError as ve: + raise ValidationError( + detail={"error": f'"{purl}" is not a valid Package URL: {ve}'}, + ) + + lookups = get_purl_query_lookups(purl) + return self.queryset.filter(**lookups) + + +class V2PackageViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Package.objects.all() + serializer_class = V2LinkedPackageSerializer + lookup_field = "purl" + filter_backends = (filters.DjangoFilterBackend,) + filterset_class = V2PackageFilterSet + throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] + + def get_queryset(self): + return super().get_queryset().with_is_vulnerable().prefetch_related("vulnerabilities") + + @action(detail=False, methods=["get"]) + def all(self, request): + """ + Return vulnerable package PURLs. + """ + vulnerable_purls = ( + Package.objects.vulnerable() + .only("package_url") + .order_by("package_url") + .distinct() + .values_list("package_url") + ) + return Response(vulnerable_purls) + + @extend_schema( + request=V2LookupRequestSerializer, + responses={200: V2PackageSerializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=V2LookupRequestSerializer, + filter_backends=[], + ) + def lookup(self, request): + """ + Return packages for a single PURL. + """ + return self._do_lookup(request, field="") + + @extend_schema( + request=V2PackageurlListSerializer, + responses={200: V2PackageSerializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=V2PackageurlListSerializer, + filter_backends=[], + ) + def bulk_lookup(self, request): + """ + Return packages for a list of PURLs. + """ + return self._do_lookup(request, field="purls") + + def _do_lookup(self, request, field): + assert field in ("purl", "purls") + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + message = ("A 'purl' or 'purls' list is required.",) + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={"error": serializer.errors, "message": message}, + ) + validated_data = serializer.validated_data + purls = validated_data.get(field) + + if field == "purl": + purls = [purls] + qs = Package.objects.for_purl(purls).with_is_vulnerable() + + return Response(V2PackageSerializer(qs, many=True, context={"request": request}).data) + + +class V2VulnerabilityFilterSet(filters.FilterSet): + class Meta: + model = Vulnerability + fields = ["vulnerability_id"] + + +class VulnerabilityViewSet(viewsets.ReadOnlyModelViewSet): + """ + Lookup for vulnerabilities by id. + """ + + queryset = Vulnerability.objects.all() + serializer_class = V2VulnerabilitySerializer + lookup_field = "vulnerability_id" + filter_backends = (filters.DjangoFilterBackend,) + filterset_class = V2VulnerabilityFilterSet + throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] + + def get_queryset(self): + """ + Assign filtered packages queryset from `get_fixed_packages_qs` + to a custom attribute `filtered_fixed_packages` + """ + return ( + super() + .get_queryset() + .prefetch_related( + "weaknesses", + "severities", + # "exploits", + ) + ) + + +class CPEFilterSet(filters.FilterSet): + cpe = filters.CharFilter(method="filter_cpe") + + def filter_cpe(self, queryset, name, value): + cpe = unquote(value) + return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct() + + +class CPEViewSet(viewsets.ReadOnlyModelViewSet): + """ + Search for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe) + """ + + queryset = Vulnerability.objects.filter( + vulnerabilityreference__reference_id__startswith="cpe" + ).distinct() + serializer_class = V2VulnerabilitySerializer + filter_backends = (filters.DjangoFilterBackend,) + throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] + filterset_class = CPEFilterSet + + @action(detail=False, methods=["post"]) + def bulk_search(self, request): + """ + Search for vulnerabilities referencing any of list of CPEs. + """ + cpes = request.data.get("cpes", []) or [] + if not cpes or not isinstance(cpes, list): + return Response( + status=400, + data={"Error": "A non-empty 'cpes' list of CPEs is required."}, + ) + for cpe in cpes: + if not cpe.startswith("cpe"): + return Response(status=400, data={"Error": f"Invalid CPE: {cpe}"}) + qs = Vulnerability.objects.filter(vulnerabilityreference__reference_id__in=cpes).distinct() + return Response(V2VulnerabilitySerializer(qs, many=True, context={"request": request}).data) + + +class AliasFilterSet(filters.FilterSet): + alias = filters.CharFilter(method="filter_alias") + + def filter_alias(self, queryset, name, value): + alias = unquote(value) + return self.queryset.filter(aliases__alias__icontains=alias) + + +class AliasViewSet(viewsets.ReadOnlyModelViewSet): + """ + Lookup for vulnerabilities by vulnerability aliases such as a CVE + (https://nvd.nist.gov/general/cve-process). + """ + + queryset = Vulnerability.objects.all() + serializer_class = V2VulnerabilitySerializer + filter_backends = (filters.DjangoFilterBackend,) + filterset_class = AliasFilterSet + throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py new file mode 100644 index 000000000..2ab782d59 --- /dev/null +++ b/vulnerabilities/api_v2.py @@ -0,0 +1,505 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +from django_filters import rest_framework as filters +from drf_spectacular.utils import OpenApiParameter +from drf_spectacular.utils import extend_schema +from drf_spectacular.utils import extend_schema_view +from packageurl import PackageURL +from rest_framework import serializers +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.reverse import reverse + +from vulnerabilities.api import PackageFilterSet +from vulnerabilities.api import VulnerabilitySeveritySerializer +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness + + +class WeaknessV2Serializer(serializers.ModelSerializer): + cwe_id = serializers.CharField() + name = serializers.CharField() + description = serializers.CharField() + + class Meta: + model = Weakness + fields = ["cwe_id", "name", "description"] + + +class VulnerabilityReferenceV2Serializer(serializers.ModelSerializer): + url = serializers.CharField() + reference_type = serializers.CharField() + reference_id = serializers.CharField() + + class Meta: + model = VulnerabilityReference + fields = ["url", "reference_type", "reference_id"] + + +class VulnerabilitySeverityV2Serializer(serializers.ModelSerializer): + class Meta: + model = VulnerabilitySeverity + fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] + + def to_representation(self, instance): + data = super().to_representation(instance) + published_at = data.get("published_at", None) + if not published_at: + data.pop("published_at") + return data + + +class VulnerabilityV2Serializer(serializers.ModelSerializer): + aliases = serializers.SerializerMethodField() + weaknesses = WeaknessV2Serializer(many=True) + references = VulnerabilityReferenceV2Serializer(many=True, source="vulnerabilityreference_set") + severities = VulnerabilitySeverityV2Serializer(many=True) + exploitability = serializers.FloatField(read_only=True) + weighted_severity = serializers.FloatField(read_only=True) + risk_score = serializers.FloatField(read_only=True) + + class Meta: + model = Vulnerability + fields = [ + "vulnerability_id", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + ] + + def get_aliases(self, obj): + return [alias.alias for alias in obj.aliases.all()] + + +class VulnerabilityListSerializer(serializers.ModelSerializer): + url = serializers.SerializerMethodField() + + class Meta: + model = Vulnerability + fields = ["vulnerability_id", "url"] + + def get_url(self, obj): + request = self.context.get("request") + return reverse( + "vulnerability-v2-detail", + kwargs={"vulnerability_id": obj.vulnerability_id}, + request=request, + ) + + +@extend_schema_view( + list=extend_schema( + parameters=[ + OpenApiParameter( + name="vulnerability_id", + description="Filter by one or more vulnerability IDs", + required=False, + type={"type": "array", "items": {"type": "string"}}, + location=OpenApiParameter.QUERY, + ), + OpenApiParameter( + name="alias", + description="Filter by alias (CVE or other unique identifier)", + required=False, + type={"type": "array", "items": {"type": "string"}}, + location=OpenApiParameter.QUERY, + ), + ] + ) +) +class VulnerabilityV2ViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Vulnerability.objects.all() + serializer_class = VulnerabilityV2Serializer + lookup_field = "vulnerability_id" + + def get_queryset(self): + queryset = super().get_queryset() + vulnerability_ids = self.request.query_params.getlist("vulnerability_id") + aliases = self.request.query_params.getlist("alias") + + if vulnerability_ids: + queryset = queryset.filter(vulnerability_id__in=vulnerability_ids) + + if aliases: + queryset = queryset.filter(aliases__alias__in=aliases).distinct() + + return queryset + + def get_serializer_class(self): + if self.action == "list": + return VulnerabilityListSerializer + return super().get_serializer_class() + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + vulnerability_ids = request.query_params.getlist("vulnerability_id") + + # If exactly one vulnerability_id is provided, return the serialized data + if len(vulnerability_ids) == 1: + try: + vulnerability = queryset.get(vulnerability_id=vulnerability_ids[0]) + serializer = self.get_serializer(vulnerability) + return Response(serializer.data) + except Vulnerability.DoesNotExist: + return Response({"detail": "Not found."}, status=404) + + # Otherwise, return a dictionary of vulnerabilities keyed by vulnerability_id + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + data = serializer.data + vulnerabilities = {item["vulnerability_id"]: item for item in data} + return self.get_paginated_response({"vulnerabilities": vulnerabilities}) + + serializer = self.get_serializer(queryset, many=True) + data = serializer.data + vulnerabilities = {item["vulnerability_id"]: item for item in data} + return Response({"vulnerabilities": vulnerabilities}) + + +class PackageV2Serializer(serializers.ModelSerializer): + purl = serializers.CharField(source="package_url") + risk_score = serializers.FloatField(read_only=True) + affected_by_vulnerabilities = serializers.SerializerMethodField() + fixing_vulnerabilities = serializers.SerializerMethodField() + next_non_vulnerable_version = serializers.CharField(read_only=True) + latest_non_vulnerable_version = serializers.CharField(read_only=True) + + class Meta: + model = Package + fields = [ + "purl", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", + "risk_score", + ] + + def get_affected_by_vulnerabilities(self, obj): + return [vuln.vulnerability_id for vuln in obj.affected_by_vulnerabilities.all()] + + def get_fixing_vulnerabilities(self, obj): + # Ghost package should not fix any vulnerability. + if obj.is_ghost: + return [] + return [vuln.vulnerability_id for vuln in obj.fixing_vulnerabilities.all()] + + +class PackageurlListSerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + allow_empty=False, + help_text="List of PackageURL strings in canonical form.", + ) + + +class PackageBulkSearchRequestSerializer(PackageurlListSerializer): + purl_only = serializers.BooleanField(required=False, default=False) + plain_purl = serializers.BooleanField(required=False, default=False) + + +class LookupRequestSerializer(serializers.Serializer): + purl = serializers.CharField( + required=True, + help_text="PackageURL strings in canonical form.", + ) + + +class PackageV2FilterSet(filters.FilterSet): + affected_by_vulnerability = filters.CharFilter( + field_name="affected_by_vulnerabilities__vulnerability_id" + ) + fixing_vulnerability = filters.CharFilter(field_name="fixing_vulnerabilities__vulnerability_id") + purl = filters.CharFilter(field_name="package_url") + + +class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Package.objects.all() + serializer_class = PackageV2Serializer + filter_backends = (filters.DjangoFilterBackend,) + filterset_class = PackageV2FilterSet + + def get_queryset(self): + queryset = super().get_queryset() + package_purls = self.request.query_params.getlist("purl") + affected_by_vulnerability = self.request.query_params.get("affected_by_vulnerability") + fixing_vulnerability = self.request.query_params.get("fixing_vulnerability") + + if package_purls: + queryset = queryset.filter(package_url__in=package_purls) + if affected_by_vulnerability: + queryset = queryset.filter( + affected_by_vulnerabilities__vulnerability_id=affected_by_vulnerability + ) + if fixing_vulnerability: + queryset = queryset.filter( + fixing_vulnerabilities__vulnerability_id=fixing_vulnerability + ) + return queryset.with_is_vulnerable() + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + + # Apply pagination + page = self.paginate_queryset(queryset) + if page is not None: + # Collect only vulnerabilities for packages in the current page + vulnerabilities = set() + for package in page: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + # Serialize the vulnerabilities with vulnerability_id as keys + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data + for vuln in vulnerabilities + } + + # Serialize the current page of packages + serializer = self.get_serializer(page, many=True) + data = serializer.data + + # Use 'self.get_paginated_response' to include pagination data + return self.get_paginated_response( + {"vulnerabilities": vulnerability_data, "packages": data} + ) + + # If pagination is not applied, collect vulnerabilities for all packages + vulnerabilities = set() + for package in queryset: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + # Serialize all packages when pagination is not applied + serializer = self.get_serializer(queryset, many=True) + data = serializer.data + return Response({"vulnerabilities": vulnerability_data, "packages": data}) + + @extend_schema( + request=PackageurlListSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=PackageurlListSerializer, + filter_backends=[], + pagination_class=None, + ) + def bulk_lookup(self, request): + """ + Return the response for exact PackageURLs requested for. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A non-empty 'purls' list of PURLs is required.", + }, + ) + validated_data = serializer.validated_data + purls = validated_data.get("purls") + + # Fetch packages matching the provided purls + packages = Package.objects.for_purls(purls).with_is_vulnerable() + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + # Serialize vulnerabilities with vulnerability_id as keys + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + # Serialize packages + package_data = PackageV2Serializer( + packages, + many=True, + context={"request": request}, + ).data + + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + @extend_schema( + request=PackageBulkSearchRequestSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=PackageBulkSearchRequestSerializer, + filter_backends=[], + pagination_class=None, + ) + def bulk_search(self, request): + """ + Lookup for vulnerable packages using many Package URLs at once. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A non-empty 'purls' list of PURLs is required.", + }, + ) + validated_data = serializer.validated_data + purls = validated_data.get("purls") + purl_only = validated_data.get("purl_only", False) + plain_purl = validated_data.get("plain_purl", False) + + if plain_purl: + purl_objects = [PackageURL.from_string(purl) for purl in purls] + plain_purl_objects = [ + PackageURL( + type=purl.type, + namespace=purl.namespace, + name=purl.name, + version=purl.version, + ) + for purl in purl_objects + ] + plain_purls = [str(purl) for purl in plain_purl_objects] + + query = ( + Package.objects.filter(plain_package_url__in=plain_purls) + .order_by("plain_package_url") + .distinct("plain_package_url") + .with_is_vulnerable() + ) + + packages = query + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data + for vuln in vulnerabilities + } + + if not purl_only: + package_data = PackageV2Serializer( + packages, many=True, context={"request": request} + ).data + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + # Using order by and distinct because there will be + # many fully qualified purl for a single plain purl + vulnerable_purls = query.vulnerable().only("plain_package_url") + vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] + return Response(data=vulnerable_purls) + + query = Package.objects.filter(package_url__in=purls).distinct().with_is_vulnerable() + packages = query + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + if not purl_only: + package_data = PackageV2Serializer( + packages, many=True, context={"request": request} + ).data + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + vulnerable_purls = query.vulnerable().only("package_url") + vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] + return Response(data=vulnerable_purls) + + @action(detail=False, methods=["get"]) + def all(self, request): + """ + Return a list of Package URLs of vulnerable packages. + """ + vulnerable_purls = ( + Package.objects.vulnerable() + .only("package_url") + .order_by("package_url") + .distinct() + .values_list("package_url", flat=True) + ) + return Response(vulnerable_purls) + + @extend_schema( + request=LookupRequestSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=LookupRequestSerializer, + filter_backends=[], + pagination_class=None, + ) + def lookup(self, request): + """ + Return the response for exact PackageURL requested for. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A 'purl' is required.", + }, + ) + validated_data = serializer.validated_data + purl = validated_data.get("purl") + + qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() + return Response(PackageV2Serializer(qs, many=True, context={"request": request}).data) diff --git a/vulnerabilities/forms.py b/vulnerabilities/forms.py index 793936c72..a00885637 100644 --- a/vulnerabilities/forms.py +++ b/vulnerabilities/forms.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 4c8e26889..0dcafda10 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -18,13 +18,13 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Importer -from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improver import Inference from vulnerabilities.improvers.default import DefaultImporter from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityReference @@ -180,54 +180,55 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver reference_id=ref.reference_id, url=ref.url, ) - if not reference: - continue - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) + if reference: + VulnerabilityRelatedReference.objects.update_or_create( + reference=reference, + vulnerability=vulnerability, + ) updated = False for severity in ref.severities: try: published_at = str(severity.published_at) if severity.published_at else None - _vs, updated = VulnerabilitySeverity.objects.update_or_create( + ( + vulnerability_severity, + updated, + ) = VulnerabilitySeverity.objects.update_or_create( scoring_system=severity.system.identifier, - reference=reference, + url=ref.url, + value=severity.value, + scoring_elements=severity.scoring_elements, defaults={ - "value": str(severity.value), - "scoring_elements": str(severity.scoring_elements), "published_at": published_at, }, ) + vulnerability.severities.add(vulnerability_severity) except: logger.error( f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}" ) if updated: logger.info( - f"Severity updated for reference {ref!r} to value: {severity.value!r} " + f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " f"and scoring_elements: {severity.scoring_elements!r}" ) for affected_purl in inference.affected_purls or []: vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) - PackageRelatedVulnerability( + AffectedByPackageRelatedVulnerability( vulnerability=vulnerability, package=vulnerable_package, created_by=improver_name, confidence=inference.confidence, - fix=False, ).update_or_create(advisory=advisory) if inference.fixed_purl: fixed_package, _ = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl) - PackageRelatedVulnerability( + FixingPackageRelatedVulnerability( vulnerability=vulnerability, package=fixed_package, created_by=improver_name, confidence=inference.confidence, - fix=True, ).update_or_create(advisory=advisory) if inference.weaknesses and vulnerability: diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 005534512..c5a5c5743 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -223,18 +223,30 @@ def from_dict(cls, affected_pkg: dict): """ package = PackageURL(**affected_pkg["package"]) affected_version_range = None - if ( - affected_pkg["affected_version_range"] - and affected_pkg["affected_version_range"] != "None" - ): - affected_version_range = VersionRange.from_string( - affected_pkg["affected_version_range"] - ) + affected_range = affected_pkg["affected_version_range"] + + # TODO: "None" is a likely bug + if affected_range and affected_range != "None": + try: + affected_version_range = VersionRange.from_string(affected_range) + except: + tb = traceback.format_exc() + logger.error( + f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}" + ) + return + fixed_version = affected_pkg["fixed_version"] if fixed_version and affected_version_range: # TODO: revisit after https://github.com/nexB/univers/issues/10 fixed_version = affected_version_range.version_class(fixed_version) + if not fixed_version and not affected_version_range: + logger.error( + f"Cannot create AffectedPackage without fixed version or affected range: {affected_pkg!r}" + ) + return + return cls( package=package, affected_version_range=affected_version_range, @@ -295,7 +307,9 @@ def from_dict(cls, advisory_data): "aliases": advisory_data["aliases"], "summary": advisory_data["summary"], "affected_packages": [ - AffectedPackage.from_dict(pkg) for pkg in advisory_data["affected_packages"] + AffectedPackage.from_dict(pkg) + for pkg in advisory_data["affected_packages"] + if pkg is not None ], "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], "date_published": datetime.datetime.fromisoformat(date_published) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 7fb4743d3..b40bdfc3a 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,26 +13,20 @@ from vulnerabilities.importers import apache_kafka from vulnerabilities.importers import apache_tomcat from vulnerabilities.importers import archlinux +from vulnerabilities.importers import curl from vulnerabilities.importers import debian from vulnerabilities.importers import debian_oval from vulnerabilities.importers import elixir_security from vulnerabilities.importers import epss from vulnerabilities.importers import fireeye from vulnerabilities.importers import gentoo -from vulnerabilities.importers import github from vulnerabilities.importers import github_osv -from vulnerabilities.importers import gitlab from vulnerabilities.importers import istio from vulnerabilities.importers import mozilla -from vulnerabilities.importers import nginx -from vulnerabilities.importers import npm -from vulnerabilities.importers import nvd from vulnerabilities.importers import openssl from vulnerabilities.importers import oss_fuzz from vulnerabilities.importers import postgresql from vulnerabilities.importers import project_kb_msr2019 -from vulnerabilities.importers import pypa -from vulnerabilities.importers import pysec from vulnerabilities.importers import redhat from vulnerabilities.importers import retiredotnet from vulnerabilities.importers import ruby @@ -41,15 +35,16 @@ from vulnerabilities.importers import ubuntu_usn from vulnerabilities.importers import vulnrichment from vulnerabilities.importers import xen +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import github_importer +from vulnerabilities.pipelines import gitlab_importer +from vulnerabilities.pipelines import nginx_importer +from vulnerabilities.pipelines import npm_importer +from vulnerabilities.pipelines import nvd_importer +from vulnerabilities.pipelines import pypa_importer +from vulnerabilities.pipelines import pysec_importer IMPORTERS_REGISTRY = [ - nvd.NVDImporter, - github.GitHubAPIImporter, - gitlab.GitLabAPIImporter, - npm.NpmImporter, - pypa.PyPaImporter, - nginx.NginxImporter, - pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, redhat.RedhatImporter, @@ -75,8 +70,19 @@ ruby.RubyImporter, github_osv.GithubOSVImporter, almalinux.AlmaImporter, + curl.CurlImporter, epss.EPSSImporter, vulnrichment.VulnrichImporter, + pypa_importer.PyPaImporterPipeline, + npm_importer.NpmImporterPipeline, + nginx_importer.NginxImporterPipeline, + gitlab_importer.GitLabImporterPipeline, + github_importer.GitHubAPIImporterPipeline, + nvd_importer.NVDImporterPipeline, + pysec_importer.PyPIImporterPipeline, ] -IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} +IMPORTERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodeBaseImporterPipeline) else x.qualified_name: x + for x in IMPORTERS_REGISTRY +} diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 9ad2a79b5..db169184e 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 10a99dd02..3dc286803 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 89de85219..27c244b2a 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 50a02a0ec..9d371ee7d 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 2e9ef6a87..640fb24dc 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/curl.py b/vulnerabilities/importers/curl.py new file mode 100644 index 000000000..a7f5e86fa --- /dev/null +++ b/vulnerabilities/importers/curl.py @@ -0,0 +1,169 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from datetime import datetime +from datetime import timezone +from typing import Iterable +from typing import Mapping + +from cwe2.database import Database +from packageurl import PackageURL +from univers.version_range import GenericVersionRange +from univers.versions import SemverVersion + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Importer +from vulnerabilities.importer import Reference +from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.severity_systems import SCORING_SYSTEMS +from vulnerabilities.utils import fetch_response +from vulnerabilities.utils import get_cwe_id +from vulnerabilities.utils import get_item + +logger = logging.getLogger(__name__) + + +class CurlImporter(Importer): + + spdx_license_expression = "curl" + license_url = "https://curl.se/docs/copyright.html" + repo_url = "https://github.com/curl/curl-www/" + importer_name = "Curl Importer" + api_url = "https://curl.se/docs/vuln.json" + + def fetch(self) -> Iterable[Mapping]: + response = fetch_response(self.api_url) + return response.json() + + def advisory_data(self) -> Iterable[AdvisoryData]: + raw_data = self.fetch() + for data in raw_data: + cve_id = data.get("aliases") or [] + cve_id = cve_id[0] if len(cve_id) > 0 else None + if not cve_id.startswith("CVE"): + package = data.get("database_specific").get("package") + logger.error(f"Invalid CVE ID: {cve_id} in package {package}") + continue + yield parse_advisory_data(data) + + +def parse_advisory_data(raw_data) -> AdvisoryData: + """ + Parse advisory data from raw JSON data and return an AdvisoryData object. + + Args: + raw_data (dict): Raw JSON data containing advisory information. + + Returns: + AdvisoryData: Parsed advisory data as an AdvisoryData object. + + Example: + >>> raw_data = { + ... "aliases": ["CVE-2024-2379"], + ... "summary": "QUIC certificate check bypass with wolfSSL", + ... "database_specific": { + ... "package": "curl", + ... "URL": "https://curl.se/docs/CVE-2024-2379.json", + ... "www": "https://curl.se/docs/CVE-2024-2379.html", + ... "issue": "https://hackerone.com/reports/2410774", + ... "severity": "Low", + ... "CWE": { + ... "id": "CWE-297", + ... "desc": "Improper Validation of Certificate with Host Mismatch" + ... }, + ... }, + ... "published": "2024-03-27T08:00:00.00Z", + ... "affected": [ + ... { + ... "ranges": [ + ... { + ... "type": "SEMVER", + ... "events": [ + ... {"introduced": "8.6.0"}, + ... {"fixed": "8.7.0"} + ... ] + ... } + ... ], + ... "versions": ["8.6.0"] + ... } + ... ] + ... } + >>> parse_advisory_data(raw_data) + AdvisoryData(aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], url='https://curl.se/docs/CVE-2024-2379.json') + """ + + affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else [] + + ranges = get_item(affected, "ranges")[0] if len(get_item(affected, "ranges")) > 0 else [] + events = get_item(ranges, "events")[1] if len(get_item(ranges, "events")) > 1 else {} + version_type = get_item(ranges, "type") if get_item(ranges, "type") else "" + fixed_version = events.get("fixed") + if version_type == "SEMVER" and fixed_version: + fixed_version = SemverVersion(fixed_version) + + purl = PackageURL(type="generic", namespace="curl.se", name="curl") + versions = affected.get("versions") or [] + affected_version_range = GenericVersionRange.from_versions(versions) + + affected_package = AffectedPackage( + package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version + ) + + database_specific = raw_data.get("database_specific") or {} + severity = VulnerabilitySeverity( + system=SCORING_SYSTEMS["cvssv3.1"], value=database_specific.get("severity", "") + ) + + references = [] + ref_www = database_specific.get("www") or "" + ref_issue = database_specific.get("issue") or "" + if ref_www: + references.append(Reference(url=ref_www, severities=[severity])) + if ref_issue: + references.append(Reference(url=ref_issue)) + + date_published = datetime.strptime( + raw_data.get("published") or "", "%Y-%m-%dT%H:%M:%S.%fZ" + ).replace(tzinfo=timezone.utc) + weaknesses = get_cwe_from_curl_advisory(raw_data) + + return AdvisoryData( + aliases=raw_data.get("aliases") or [], + summary=raw_data.get("summary") or "", + affected_packages=[affected_package], + references=references, + date_published=date_published, + weaknesses=weaknesses, + url=raw_data.get("database_specific", {}).get("URL", ""), + ) + + +def get_cwe_from_curl_advisory(raw_data): + """ + Extracts CWE IDs from the given raw_data and returns a list of CWE IDs. + + >>> get_cwe_from_curl_advisory({"database_specific": {"CWE": {"id": "CWE-333"}}}) + [333] + >>> get_cwe_from_curl_advisory({"database_specific": {"CWE": {"id": ""}}}) + [] + """ + weaknesses = [] + db = Database() + cwe_string = get_item(raw_data, "database_specific", "CWE", "id") or "" + + if cwe_string: + cwe_id = get_cwe_id(cwe_string) + try: + db.get(cwe_id) + weaknesses.append(cwe_id) + except Exception: + logger.error("Invalid CWE id") + return weaknesses diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index 94057675f..e29c9b788 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py index aa3d6917d..f5a747a11 100644 --- a/vulnerabilities/importers/debian_oval.py +++ b/vulnerabilities/importers/debian_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 4fd492a92..3fe0ec15b 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import urllib.parse as urlparse diff --git a/vulnerabilities/importers/epss.py b/vulnerabilities/importers/epss.py index 83822fa5d..982229e09 100644 --- a/vulnerabilities/importers/epss.py +++ b/vulnerabilities/importers/epss.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import csv diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 453afe10d..34daf97e0 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 2c91f7f2f..2f569cdf1 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/github_osv.py b/vulnerabilities/importers/github_osv.py index bef06a8af..f0490044e 100644 --- a/vulnerabilities/importers/github_osv.py +++ b/vulnerabilities/importers/github_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/importers/gsd.py b/vulnerabilities/importers/gsd.py new file mode 100644 index 000000000..4d69bd63a --- /dev/null +++ b/vulnerabilities/importers/gsd.py @@ -0,0 +1,222 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +import logging +from io import BytesIO +from typing import Iterable +from typing import List +from typing import Set +from zipfile import ZipFile + +import dateparser +import requests + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import Importer +from vulnerabilities.importer import Reference +from vulnerabilities.utils import build_description +from vulnerabilities.utils import dedupe + +logger = logging.getLogger(__name__) + + +class GSDImporter: # TODO inherit from Importer + spdx_license_expression = "CC0-1.0" + license_url = "https://github.com/cloudsecurityalliance/gsd-database/blob/main/LICENSE" + url = "https://codeload.github.com/cloudsecurityalliance/gsd-database/zip/refs/heads/main" + + def advisory_data(self) -> Iterable[AdvisoryData]: + response = requests.get(self.url).content + with ZipFile(BytesIO(response)) as zip_file: + for file_name in zip_file.namelist(): + if file_name == "gsd-database-main/allowlist.json" or not file_name.endswith( + ".json" + ): + continue + + with zip_file.open(file_name) as f: + try: + raw_data = json.load(f) + yield parse_advisory_data(raw_data, file_name) + except Exception as e: + logger.error(f"Invalid GSD advisory data file: {file_name} - {e}") + + +def parse_advisory_data(raw_data, file_name): + """ + Parse a GSD advisory file and return an AdvisoryData. + Each advisory file contains the advisory information in JSON format. + """ + + namespaces = raw_data.get("namespaces") or {} + cve_org = namespaces.get("cve.org") or {} + nvd_nist_gov = namespaces.get("nvd.nist.gov") or {} + + gsd = raw_data.get("GSD") or {} + gsd_id = gsd.get("id") or file_name + gsd_alias = gsd.get("alias") or [] + gsd_description = gsd.get("description") or "" + + gsd_reference_data = gsd.get("") or [] + gsd_references = [Reference(url=ref) for ref in gsd_reference_data] + + details = gsd_description or "".join(get_description(cve_org)) + + aliases_cve_org = get_aliases(cve_org) + aliases_nvd_nist_gov = get_aliases(nvd_nist_gov) + + aliases = [gsd_alias, gsd_id] + aliases_cve_org + aliases_nvd_nist_gov + aliases = [alias for alias in aliases if alias is not None] + + summary = build_description(summary=get_summary(cve_org), description=details) + + severities = get_severities(cve_org) + configurations = nvd_nist_gov.get("configurations") or {} + nodes = configurations.get("nodes") or [] + cpes = get_cpe(nodes) + + references = get_references(cve_org) + gsd_references + + date_published = get_published_date_nvd_nist_gov(nvd_nist_gov) + + return AdvisoryData( + aliases=dedupe(aliases), + summary=summary, + references=references, + date_published=date_published, + ) + + +def get_summary(cve) -> str: + """ + Returns a title of CVE_data_meta + >> get_summary {"CVE_data_meta": {"TITLE": "DoS vulnerability: Invalid Accent Colors"} + 'DoS vulnerability: Invalid Accent Colors' + """ + cve_data_meta = cve.get("CVE_data_meta") or {} + return cve_data_meta.get("TITLE") or "" + + +def get_severities(cve) -> List: + """ + Return a list of CVSS vectorString + >>> get_severities({"impact": {"cvss": {"vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H"}}}) + ['CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H'] + """ + severities = [] + impact = cve.get("impact") or {} + + base_metric_2 = impact.get("baseMetricV2") or {} + if base_metric_2: + cvss_v2 = base_metric_2.get("cvssV2") or {} + cvss_vector = cvss_v2.get("vectorString") + if cvss_vector: + severities.append(cvss_vector) + + base_metric_v3 = impact.get("baseMetricV3") or {} + if base_metric_v3: + cvss_v3 = base_metric_v3.get("cvssV3") or {} + cvss_vector = cvss_v3.get("vectorString") + if cvss_vector: + severities.append(cvss_vector) + + cvss = impact.get("cvss") or {} + if isinstance(cvss, List): + for cvss_v in cvss: + if isinstance(cvss_v, dict): + cvss_vector = cvss_v.get("vectorString") or {} + if cvss_vector: + severities.append(cvss_vector) + else: + cvss_vector = cvss.get("vectorString") + if cvss_vector: + severities.append(cvss_vector) + return severities + + +def get_description(cve) -> [str]: + """ + Get a list description value from description object + >>> get_description({"description": {"description_data": [{"lang": "eng","value": "the description"}]}}) + ['the description'] + """ + description = cve.get("description") or {} + description_data = description.get("description_data") or [] + return [desc["value"] for desc in description_data if desc["value"] and desc["lang"] == "eng"] + + +def get_references(cve): + """ + Returns a list of Reference assigned with url + >>> get_references({"references": { + ... "reference_data": [{ + ... "name": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", + ... "refsource": "CONFIRM", + ... "tags": ["Vendor Advisory"], + ... "url": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198"}]}}) + [Reference(reference_id='', reference_type='', url='https://kc.mcafee.com/corporate/index?page=content&id=SB10198', severities=[])] + """ + references = cve.get("references") or {} + reference_data = references.get("reference_data") or [] + return [Reference(url=ref["url"]) for ref in reference_data if ref["url"]] + + +def get_aliases(cve) -> [str]: + """ + Returns a list of aliases + >>> get_aliases({"CVE_data_meta": {"ID": "CVE-2017-4017"},"source": {"advisory": "GHSA-v8x6-59g4-5g3w"}}) + ['CVE-2017-4017', 'GHSA-v8x6-59g4-5g3w'] + """ + cve_data_meta = cve.get("CVE_data_meta") or {} + alias = cve_data_meta.get("ID") + + source = cve.get("source") or {} + advisory = source.get("advisory") + + aliases = [] + if alias: + aliases.append(alias) + if advisory: + aliases.append(advisory) + return aliases + + +def get_published_date_nvd_nist_gov(nvd_nist_gov): + """ + Returns a published datetime + >>> get_published_date_nvd_nist_gov({"publishedDate": "2022-06-23T07:15Z"}) + datetime.datetime(2022, 6, 23, 7, 15, tzinfo=) + """ + published_date = nvd_nist_gov.get("publishedDate") + return published_date and dateparser.parse(published_date) + + +def get_cpe(nodes) -> List: + """ + >>> get_cpe([{"children": [], "cpe_match": [{ + ... "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", + ... "cpe_name": [], + ... "versionEndIncluding": "1.2.5.1", + ... "vulnerable": True + ... },{ + ... "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", + ... "cpe_name": [], + ... "versionEndIncluding": "1.3.25", + ... "vulnerable": True + ... }],"operator": "OR"}]) + ['cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*', 'cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*'] + """ + cpe_list = [] + for node in nodes: + cpe_match = node.get("cpe_match") or [] + for cpe23Uri in cpe_match: + cpe_uri = cpe23Uri.get("cpe23Uri") + if cpe_uri: + cpe_list.append(cpe_uri) + return cpe_list diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9341a76de..8f9f6334a 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/kaybee.py b/vulnerabilities/importers/kaybee.py index 1b908e4b5..ccee4a68f 100644 --- a/vulnerabilities/importers/kaybee.py +++ b/vulnerabilities/importers/kaybee.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py index 62eddbeb0..a422ea32a 100644 --- a/vulnerabilities/importers/mattermost.py +++ b/vulnerabilities/importers/mattermost.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 11667badc..8eea10370 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index ca69436c9..b71206418 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/oss_fuzz.py b/vulnerabilities/importers/oss_fuzz.py index e81f06fc2..63b879990 100644 --- a/vulnerabilities/importers/oss_fuzz.py +++ b/vulnerabilities/importers/oss_fuzz.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py index 1ee45a1be..90f4200e8 100644 --- a/vulnerabilities/importers/osv.py +++ b/vulnerabilities/importers/osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index ee6de3976..70ab1bfe9 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index e099d3f36..a006b1353 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/pypa.py b/vulnerabilities/importers/pypa.py deleted file mode 100644 index e0648e1c2..000000000 --- a/vulnerabilities/importers/pypa.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import logging -import os -from pathlib import Path -from typing import Iterable - -import saneyaml - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.utils import get_advisory_url - -logger = logging.getLogger(__name__) - - -class PyPaImporter(Importer): - license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" - spdx_license_expression = "CC-BY-4.0" - repo_url = "git+https://github.com/pypa/advisory-database" - importer_name = "Pypa Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - vcs_response = self.clone(repo_url=self.repo_url) - path = Path(vcs_response.dest_dir) - for advisory_url, raw_data in fork_and_get_files(base_path=path): - yield parse_advisory_data( - raw_data=raw_data, - supported_ecosystems=["pypi"], - advisory_url=advisory_url, - ) - finally: - if self.vcs_response: - self.vcs_response.delete() - - -class ForkError(Exception): - pass - - -def fork_and_get_files(base_path) -> dict: - """ - Yield advisorie data mappings from the PyPA GitHub repository at ``url``. - """ - advisory_dirs = os.path.join(base_path, "vulns") - for root, _, files in os.walk(advisory_dirs): - for file in files: - path = os.path.join(root, file) - if not file.endswith(".yaml"): - logger.warning(f"Unsupported non-YAML PyPA advisory file: {path}") - continue - advisory_url = get_advisory_url( - file=Path(path), - base_path=base_path, - url="https://github.com/pypa/advisory-database/blob/main/", - ) - with open(path) as f: - yield advisory_url, saneyaml.load(f.read()) diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py deleted file mode 100644 index 058747463..000000000 --- a/vulnerabilities/importers/pysec.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import logging -from io import BytesIO -from typing import Iterable -from zipfile import ZipFile - -import requests - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importers.osv import parse_advisory_data - -logger = logging.getLogger(__name__) - - -class PyPIImporter(Importer): - license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" - spdx_license_expression = "CC-BY-4.0" - importer_name = "PyPI Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - """ - Yield AdvisoryData using a zipped data dump of OSV data - """ - url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" - response = requests.get(url).content - with ZipFile(BytesIO(response)) as zip_file: - for file_name in zip_file.namelist(): - if not file_name.startswith("PYSEC-"): - logger.error(f"Unsupported PyPI advisory data file: {file_name}") - continue - with zip_file.open(file_name) as f: - vul_info = json.load(f) - yield parse_advisory_data( - raw_data=vul_info, supported_ecosystems=["pypi"], advisory_url=url - ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index a2cc1940b..68e3d5062 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index 0ed580ba1..139ecd1af 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 6a3b5f3f1..268419587 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index a1e97c277..c61907a82 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_backports.py b/vulnerabilities/importers/suse_backports.py index 6d661ef3f..e7863e7e7 100644 --- a/vulnerabilities/importers/suse_backports.py +++ b/vulnerabilities/importers/suse_backports.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_oval.py b/vulnerabilities/importers/suse_oval.py index 5ac35e23e..0722682f7 100644 --- a/vulnerabilities/importers/suse_oval.py +++ b/vulnerabilities/importers/suse_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index f43e69576..b7f2089ac 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 646b40028..e47515b93 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 6bef117f9..1aa247ec6 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/vulnrichment.py b/vulnerabilities/importers/vulnrichment.py index 15a30e01e..9eb4d3bcb 100644 --- a/vulnerabilities/importers/vulnrichment.py +++ b/vulnerabilities/importers/vulnrichment.py @@ -181,7 +181,7 @@ def parse_cve_advisory(raw_data, advisory_url): summary=summary, references=references, date_published=date_published, - weaknesses=list(weaknesses), + weaknesses=sorted(weaknesses), url=advisory_url, ) diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 2b28a9771..a0cafa324 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index cb8fb6a04..452e1e2f6 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -17,9 +17,10 @@ from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityReference @@ -135,12 +136,11 @@ def process_inferences( vulnerable_package, created = Package.objects.get_or_create_from_purl( purl=affected_purl ) - PackageRelatedVulnerability( + AffectedByPackageRelatedVulnerability( vulnerability=vulnerability, package=vulnerable_package, created_by=improver_name, confidence=inference.confidence, - fix=False, ).update_or_create( advisory=advisory, ) @@ -149,12 +149,11 @@ def process_inferences( fixed_package, created = Package.objects.get_or_create_from_purl( purl=inference.fixed_purl ) - PackageRelatedVulnerability( + FixingPackageRelatedVulnerability( vulnerability=vulnerability, package=fixed_package, created_by=improver_name, confidence=inference.confidence, - fix=True, ).update_or_create( advisory=advisory, ) diff --git a/vulnerabilities/improver.py b/vulnerabilities/improver.py index b8caeb9ed..0d5e7f13d 100644 --- a/vulnerabilities/improver.py +++ b/vulnerabilities/improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 38cad4243..393ffb234 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -3,13 +3,19 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from vulnerabilities.improvers import valid_versions -from vulnerabilities.improvers import vulnerability_kev from vulnerabilities.improvers import vulnerability_status +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipelines import compute_package_risk +from vulnerabilities.pipelines import compute_package_version_rank +from vulnerabilities.pipelines import enhance_with_exploitdb +from vulnerabilities.pipelines import enhance_with_kev +from vulnerabilities.pipelines import enhance_with_metasploit +from vulnerabilities.pipelines import flag_ghost_packages IMPROVERS_REGISTRY = [ valid_versions.GitHubBasicImprover, @@ -29,7 +35,16 @@ valid_versions.GithubOSVImprover, vulnerability_status.VulnerabilityStatusImprover, valid_versions.AlmaImprover, - vulnerability_kev.VulnerabilityKevImprover, + valid_versions.CurlImprover, + flag_ghost_packages.FlagGhostPackagePipeline, + enhance_with_kev.VulnerabilityKevPipeline, + enhance_with_metasploit.MetasploitImproverPipeline, + enhance_with_exploitdb.ExploitDBImproverPipeline, + compute_package_risk.ComputePackageRiskPipeline, + compute_package_version_rank.ComputeVersionRankPipeline, ] -IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} +IMPROVERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodePipeline) else x.qualified_name: x + for x in IMPROVERS_REGISTRY +} diff --git a/vulnerabilities/improvers/add_missing_refid.py b/vulnerabilities/improvers/add_missing_refid.py index 80e7f6eeb..d3d5efa72 100644 --- a/vulnerabilities/improvers/add_missing_refid.py +++ b/vulnerabilities/improvers/add_missing_refid.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 720a143e8..f2e9009e8 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -105,6 +105,8 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL] ... ) >>> assert expected == got """ + if not affected_package: + return [], [] try: vr = affected_package.affected_version_range diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 5e5bf0dc4..9df5a54ed 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -12,7 +12,6 @@ from datetime import datetime from typing import Iterable from typing import List -from typing import Mapping from typing import Optional from django.db.models import Q @@ -29,15 +28,12 @@ from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter +from vulnerabilities.importers.curl import CurlImporter from vulnerabilities.importers.debian import DebianImporter from vulnerabilities.importers.debian_oval import DebianOvalImporter from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.importers.github import GitHubAPIImporter from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.gitlab import GitLabAPIImporter from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.importers.nginx import NginxImporter -from vulnerabilities.importers.npm import NpmImporter from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter from vulnerabilities.importers.ruby import RubyImporter from vulnerabilities.importers.ubuntu import UbuntuImporter @@ -45,6 +41,11 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline +from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline +from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline +from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage from vulnerabilities.utils import clean_nginx_git_tag from vulnerabilities.utils import get_affected_packages_by_patched_package @@ -63,6 +64,8 @@ class ValidVersionImprover(Improver): @property def interesting_advisories(self) -> QuerySet: + if issubclass(self.importer, VulnerableCodeBaseImporterPipeline): + return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).paginated() return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated() def get_package_versions( @@ -220,7 +223,7 @@ class NginxBasicImprover(Improver): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NginxImporter.qualified_name).paginated() + return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).paginated() def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: all_versions = list(self.fetch_nginx_version_from_git_tags()) @@ -364,12 +367,12 @@ class DebianBasicImprover(ValidVersionImprover): class GitLabBasicImprover(ValidVersionImprover): - importer = GitLabAPIImporter + importer = GitLabImporterPipeline ignorable_versions = [] class GitHubBasicImprover(ValidVersionImprover): - importer = GitHubAPIImporter + importer = GitHubAPIImporterPipeline ignorable_versions = frozenset( [ "0.1-bulbasaur", @@ -431,12 +434,13 @@ class GitHubBasicImprover(ValidVersionImprover): "3.0.0b3-", "3.0b6dev-r41684", "-class.-jw.util.version.Version-", + "vulnerabilities", ] ) class NpmImprover(ValidVersionImprover): - importer = NpmImporter + importer = NpmImporterPipeline ignorable_versions = [] @@ -473,8 +477,10 @@ class RubyImprover(ValidVersionImprover): class GithubOSVImprover(ValidVersionImprover): importer = GithubOSVImporter ignorable_versions = [] - - + class AlmaImprover(ValidVersionImprover): importer = AlmaImporter + +class CurlImprover(ValidVersionImprover): + importer = CurlImporter ignorable_versions = [] diff --git a/vulnerabilities/improvers/vulnerability_kev.py b/vulnerabilities/improvers/vulnerability_kev.py deleted file mode 100644 index 06e6c0380..000000000 --- a/vulnerabilities/improvers/vulnerability_kev.py +++ /dev/null @@ -1,66 +0,0 @@ -import logging -from typing import Iterable - -from django.db.models import QuerySet -from sphinx.util import requests - -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Kev - -logger = logging.getLogger(__name__) - - -class VulnerabilityKevImprover(Improver): - """ - Known Exploited Vulnerabilities Improver - """ - - @property - def interesting_advisories(self) -> QuerySet: - # TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory - return [Advisory.objects.first()] - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - """ - Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update - the Kev instance accordingly. - """ - - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) - response = requests.get(kev_url) - kev_data = response.json() - if response.status_code != 200: - logger.error( - f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}" - ) - return [] - - for kev_vul in kev_data.get("vulnerabilities", []): - alias = Alias.objects.get_or_none(alias=kev_vul["cveID"]) - if not alias: - continue - - vul = alias.vulnerability - - if not vul: - continue - - Kev.objects.update_or_create( - vulnerability=vul, - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "resources_and_notes": kev_vul["notes"], - "known_ransomware_campaign_use": True - if kev_vul["knownRansomwareCampaignUse"] == "Known" - else False, - }, - ) - return [] diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py index 0157db557..214e6dc35 100644 --- a/vulnerabilities/improvers/vulnerability_status.py +++ b/vulnerabilities/improvers/vulnerability_status.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -14,7 +14,6 @@ from django.db.models.query import QuerySet from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.nvd import NVDImporter from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory @@ -22,6 +21,7 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline from vulnerabilities.utils import fetch_response from vulnerabilities.utils import get_item @@ -38,7 +38,7 @@ class VulnerabilityStatusImprover(Improver): @property def interesting_advisories(self) -> QuerySet: return ( - Advisory.objects.filter(Q(created_by=NVDImporter.qualified_name)) + Advisory.objects.filter(Q(created_by=NVDImporterPipeline.pipeline_id)) .distinct("aliases") .paginated() ) diff --git a/vulnerabilities/lib_oval.py b/vulnerabilities/lib_oval.py index 0b178d47c..30bc7f0c0 100644 --- a/vulnerabilities/lib_oval.py +++ b/vulnerabilities/lib_oval.py @@ -5,7 +5,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/management/commands/commit_export.py b/vulnerabilities/management/commands/commit_export.py new file mode 100644 index 000000000..9d47904f3 --- /dev/null +++ b/vulnerabilities/management/commands/commit_export.py @@ -0,0 +1,179 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +import os +import shutil +import tempfile +import textwrap +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import requests +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError +from git import Repo + +from vulnerablecode.settings import ALLOWED_HOSTS +from vulnerablecode.settings import VULNERABLECODE_VERSION + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = """Commit the exported vulnerability data in the backing GitHub repository. + + This command takes the path to the exported vulnerability data and creates a pull + request in the backing GitHub repository with the changes. + """ + + def add_arguments(self, parser): + parser.add_argument( + "path", + help="Path to exported data.", + ) + + def handle(self, *args, **options): + if path := options["path"]: + base_path = Path(path) + + if not path or not base_path.is_dir(): + raise CommandError("Enter a valid directory path to the exported data.") + + vcio_export_repo_url = os.environ.get("VULNERABLECODE_EXPORT_REPO_URL") + vcio_github_service_token = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_TOKEN") + vcio_github_service_name = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_NAME") + vcio_github_service_email = os.environ.get("VULNERABLECODE_GITHUB_SERVICE_EMAIL") + + # Check for missing environment variables + missing_vars = [] + if not vcio_export_repo_url: + missing_vars.append("VULNERABLECODE_EXPORT_REPO_URL") + if not vcio_github_service_token: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_TOKEN") + if not vcio_github_service_name: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_NAME") + if not vcio_github_service_email: + missing_vars.append("VULNERABLECODE_GITHUB_SERVICE_EMAIL") + + if missing_vars: + raise CommandError(f'Missing environment variables: {", ".join(missing_vars)}') + + local_dir = tempfile.mkdtemp() + current_date = datetime.now().strftime("%Y-%m-%d") + + branch_name = f"export-update-{current_date}" + pr_title = "Update package vulnerabilities from VulnerableCode" + pr_body = f"""\ + Tool: pkg:github/aboutcode-org/vulnerablecode@v{VULNERABLECODE_VERSION} + Reference: https://{ALLOWED_HOSTS[0]}/ + """ + commit_message = f"""\ + Update package vulnerabilities from VulnerableCode + + Tool: pkg:github/aboutcode-org/vulnerablecode@v{VULNERABLECODE_VERSION} + Reference: https://{ALLOWED_HOSTS[0]}/ + + Signed-off-by: {vcio_github_service_name} <{vcio_github_service_email}> + """ + + self.stdout.write("Committing VulnerableCode package and vulnerability data.") + repo = self.clone_repository( + repo_url=vcio_export_repo_url, + local_path=local_dir, + token=vcio_github_service_token, + ) + + repo.config_writer().set_value("user", "name", vcio_github_service_name).release() + repo.config_writer().set_value("user", "email", vcio_github_service_email).release() + + self.add_changes(repo=repo, content_path=path) + + if self.commit_and_push_changes( + repo=repo, + branch=branch_name, + commit_message=textwrap.dedent(commit_message), + ): + self.create_pull_request( + repo_url=vcio_export_repo_url, + branch=branch_name, + title=pr_title, + body=textwrap.dedent(pr_body), + token=vcio_github_service_token, + ) + shutil.rmtree(local_dir) + + def clone_repository(self, repo_url, local_path, token): + """Clone repository to local_path.""" + + if os.path.exists(local_path): + shutil.rmtree(local_path) + + authenticated_repo_url = repo_url.replace("https://", f"https://{token}@") + return Repo.clone_from(authenticated_repo_url, local_path) + + def add_changes(self, repo, content_path): + """Copy changes from the ``content_path`` to ``repo``.""" + + source_path = Path(content_path) + destination_path = Path(repo.working_dir) + + for item in source_path.iterdir(): + if not item.is_dir(): + continue + target_item = destination_path / item.name + if target_item.exists(): + shutil.rmtree(target_item) + shutil.copytree(item, target_item) + + def commit_and_push_changes(self, repo, branch, commit_message, remote_name="origin"): + """Commit changes and push to remote repository, return name of changed files.""" + + repo.git.checkout("HEAD", b=branch) + files_changed = repo.git.diff("HEAD", name_only=True) + + if not files_changed: + self.stderr.write(self.style.SUCCESS("No changes to commit.")) + return + + repo.git.add(A=True) + repo.index.commit(commit_message) + repo.git.push(remote_name, branch) + return files_changed + + def create_pull_request(self, repo_url, branch, title, body, token): + """Create a pull request in the GitHub repository.""" + + url_parts = urlparse(repo_url).path + path_parts = url_parts.strip("/").rstrip(".git").split("/") + + if len(path_parts) >= 2: + repo_owner = path_parts[0] + repo_name = path_parts[1] + else: + raise ValueError("Invalid GitHub repo URL") + + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/pulls" + headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json"} + data = {"title": title, "head": branch, "base": "main", "body": body} + + response = requests.post(url, headers=headers, json=data) + + if response.status_code == 201: + pr_response = response.json() + self.stdout.write( + self.style.SUCCESS( + f"Pull request created successfully: {pr_response.get('html_url')}." + ) + ) + else: + self.stderr.write( + self.style.ERROR(f"Failed to create pull request: {response.content}") + ) diff --git a/vulnerabilities/management/commands/create_api_user.py b/vulnerabilities/management/commands/create_api_user.py index db471cac4..86f6bb486 100644 --- a/vulnerabilities/management/commands/create_api_user.py +++ b/vulnerabilities/management/commands/create_api_user.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/management/commands/export.py b/vulnerabilities/management/commands/export.py index 6f38d7838..08685e33d 100644 --- a/vulnerabilities/management/commands/export.py +++ b/vulnerabilities/management/commands/export.py @@ -3,12 +3,11 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging -import os -from hashlib import sha512 +from itertools import groupby from pathlib import Path import saneyaml @@ -16,118 +15,163 @@ from django.core.management.base import CommandError from packageurl import PackageURL +from aboutcode import hashid from vulnerabilities.models import Package logger = logging.getLogger(__name__) +def serialize_severity(sev): + return { + "score": sev.value, + "scoring_system": sev.scoring_system, + "scoring_elements": sev.scoring_elements, + "published_at": sev.published_at, + "url": sev.url, + } + + +def serialize_vulnerability(vuln): + """ + Return a plain data mapping seralized from ``vuln`` Vulnerability instance. + """ + aliases = list(vuln.aliases.values_list("alias", flat=True)) + severities = [serialize_severity(sev) for sev in vuln.severities.all()] + weaknesses = [wkns.cwe for wkns in vuln.weaknesses.all()] + + references = list( + vuln.references.values( + "url", + "reference_type", + "reference_id", + ) + ) + + return { + "vulnerability_id": vuln.vcid, + "aliases": aliases, + "summary": vuln.summary, + "severities": severities, + "weaknesses": weaknesses, + "references": references, + } + + class Command(BaseCommand): - help = "export vulnerablecode data" + help = """Export vulnerability and package data as YAML for use in FederatedCode + + This command exports the data in a tree of directories and YAML files designed such that + it is possible to access directly a vulnerability data file by only knowing its VCID, and that + it is possible to access directly the package data files by only knowing its PURL. + """ def add_arguments(self, parser): - parser.add_argument("path") + parser.add_argument( + "path", + help="Path to a directory where to export data.", + ) def handle(self, *args, **options): - if options["path"]: - git_path = Path(options["path"]) - if not git_path.is_dir(): - raise CommandError("Please enter a valid path") + if path := options["path"]: + base_path = Path(path) - self.export_data(git_path) + if not path or not base_path.is_dir(): + raise CommandError("Enter a valid directory path") - self.stdout.write(self.style.SUCCESS("Successfully exported vulnerablecode data")) + self.stdout.write("Exporting vulnerablecode Package and Vulnerability data.") + self.export_data(base_path) + self.stdout.write(self.style.SUCCESS(f"Successfully exported data to {base_path}.")) - def export_data(self, git_path): + def export_data(self, base_path: Path): """ - export vulnerablecode data - by running `python manage.py export /path/vulnerablecode-data` + Export vulnerablecode data to ``base_path``.` """ - self.stdout.write("Exporting vulnerablecode data") - - ecosystems = [pkg.type for pkg in Package.objects.distinct("type")] - - for ecosystem in ecosystems: - package_files = {} # {"package path": "data" } - vul_files = {} # {"vulnerability path": "data" } - - for purl in ( - Package.objects.filter(type=ecosystem) - .prefetch_related("vulnerabilities") - .paginated() - ): - purl_without_version = PackageURL( - type=purl.type, - namespace=purl.namespace, - name=purl.name, - ) - - # ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml - pkg_filepath = ( - f"./aboutcode-packages-{get_purl_hash(purl_without_version)}/{purl.type}/{purl.namespace}/{purl.name}" - f"/versions/vulnerabilities.yml" - ) - - package_data = { - "purl": str(purl), - "affected_by_vulnerabilities": [ - vuln.vulnerability_id for vuln in purl.affected_by - ], - "fixing_vulnerabilities": [vuln.vulnerability_id for vuln in purl.fixing], - } - - if pkg_filepath in package_files: - package_files[pkg_filepath]["versions"].append(package_data) - else: - package_files[pkg_filepath] = { - "package": str(purl_without_version), - "versions": [package_data], + i = 0 + seen_vcid = set() + + for i, (purl_without_version, package_versions) in enumerate(packages_by_type_ns_name(), 1): + pkg_version = None + try: + package_urls = [] + package_vulnerabilities = [] + for pkg_version in package_versions: + purl = pkg_version.package_url + package_urls.append(purl) + package_data = { + "purl": purl, + "affected_by_vulnerabilities": list( + pkg_version.affected_by.values_list("vulnerability_id", flat=True) + ), + "fixing_vulnerabilities": list( + pkg_version.fixing.values_list("vulnerability_id", flat=True) + ), } + package_vulnerabilities.append(package_data) - for vul in purl.vulnerabilities.all(): - vulnerability_id = vul.vulnerability_id - # ./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml - vul_filepath = ( - f"./aboutcode-vulnerabilities-{vulnerability_id[5:7]}/{vulnerability_id[10:12]}" - f"/{vulnerability_id}/{vulnerability_id}.yml" - ) - vul_files[vul_filepath] = { - "vulnerability_id": vul.vulnerability_id, - "aliases": [alias.alias for alias in vul.get_aliases], - "summary": vul.summary, - "severities": [severity for severity in vul.severities.values()], - "references": [ref for ref in vul.references.values()], - "weaknesses": [ - "CWE-" + str(weakness["cwe_id"]) for weakness in vul.weaknesses.values() - ], - } + for vuln in pkg_version.vulnerabilities: + vcid = vuln.vulnerability_id + # do not write twice the same file + if vcid in seen_vcid: + continue + + seen_vcid.add(vcid) + vulnerability = serialize_vulnerability(vuln) + vpath = hashid.get_vcid_yml_file_path(vcid) + write_file(base_path=base_path, file_path=vpath, data=vulnerability) + if (lv := len(seen_vcid)) % 100 == 0: + self.stdout.write(f"Processed {lv} vulnerabilities. Last VCID: {vcid}") + + ppath = hashid.get_package_purls_yml_file_path(purl) + write_file(base_path=base_path, file_path=ppath, data=package_urls) + + pvpath = hashid.get_package_vulnerabilities_yml_file_path(purl) + write_file(base_path=base_path, file_path=pvpath, data=package_vulnerabilities) - for items in [package_files, vul_files]: - for filepath, data in items.items(): - create_file(filepath, git_path, data) + if i % 100 == 0: + self.stdout.write(f"Processed {i} package. Last PURL: {purl_without_version}") - self.stdout.write(f"Successfully exported {ecosystem} data") + except Exception as e: + raise Exception(f"Failed to process Package: {pkg_version}") from e + self.stdout.write(f"Exported data for: {i} package and {len(seen_vcid)} vulnerabilities.") -def create_file(filepath, git_path, data): + +def by_purl_type_ns_name(package): """ - Check if the directories exist if it doesn't exist create a new one then Create the file - ./aboutcode-vulnerabilities-12/34/VCID-1223-3434-34343/VCID-1223-3434-34343.yml - ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/vulnerabilities.yml - ./aboutcode-packages-ed5/maven/org.apache.log4j/log4j-core/versions/1.2.3/vulnerabilities.yml + Key function to sort packages by type, namespace and name """ - filepath = git_path.joinpath(filepath) - dirname = os.path.dirname(filepath) - os.makedirs(dirname, exist_ok=True) - data = saneyaml.dump(data) - with open(filepath, encoding="utf-8", mode="w") as f: - f.write(data) + return package.type, package.namespace, package.name -def get_purl_hash(purl: PackageURL, length: int = 3) -> str: +def packages_by_type_ns_name(): + """ + Return a two-level iterator over all Packages grouped-by package, ignoring version. + """ + qs = ( + Package.objects.order_by("type", "namespace", "name", "version") + .prefetch_related( + "affected_by_vulnerabilities", + "affected_by_vulnerabilities__references", + "affected_by_vulnerabilities__weaknesses", + "affected_by_vulnerabilities__severities", + "fixing_vulnerabilities", + "fixing_vulnerabilities__references", + "fixing_vulnerabilities__weaknesses", + "fixing_vulnerabilities__severities", + ) + .paginated() + ) + + for tp_ns_name, packages in groupby(qs, key=by_purl_type_ns_name): + yield PackageURL(*tp_ns_name), packages + + +def write_file(base_path: Path, file_path: Path, data: dict): """ - Return a short lower cased hash of a purl. - https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154 + Write the ``data`` as YAML to the ``file_path`` in the ``base_path`` root directory. + Create directories in the path as needed. """ - purl_bytes = str(purl).encode("utf-8") - short_hash = sha512(purl_bytes).hexdigest()[:length] - return short_hash.lower() + write_to = base_path / file_path + write_to.parent.mkdir(parents=True, exist_ok=True) + with open(write_to, encoding="utf-8", mode="w") as f: + f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 5ae885299..f4876b11a 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import traceback @@ -13,6 +13,7 @@ from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers import IMPORTERS_REGISTRY +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline class Command(BaseCommand): @@ -56,6 +57,14 @@ def import_data(self, importers): failed_importers = [] for importer in importers: + if issubclass(importer, VulnerableCodeBaseImporterPipeline): + self.stdout.write(f"Importing data using {importer.pipeline_id}") + status, error = importer().execute() + if status != 0: + self.stdout.write(error) + failed_importers.append(importer.pipeline_id) + continue + self.stdout.write(f"Importing data using {importer.qualified_name}") try: ImportRunner(importer).run() diff --git a/vulnerabilities/management/commands/improve.py b/vulnerabilities/management/commands/improve.py index e14c2bacc..10ba07a27 100644 --- a/vulnerabilities/management/commands/improve.py +++ b/vulnerabilities/management/commands/improve.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -14,6 +14,7 @@ from vulnerabilities.improve_runner import ImproveRunner from vulnerabilities.improvers import IMPROVERS_REGISTRY +from vulnerabilities.pipelines import VulnerableCodePipeline class Command(BaseCommand): @@ -55,6 +56,14 @@ def improve_data(self, improvers): failed_improvers = [] for improver in improvers: + if issubclass(improver, VulnerableCodePipeline): + self.stdout.write(f"Improving data using {improver.pipeline_id}") + status, error = improver().execute() + if status != 0: + self.stdout.write(error) + failed_improvers.append(improver.pipeline_id) + continue + self.stdout.write(f"Improving data using {improver.qualified_name}") try: ImproveRunner(improver_class=improver).run() diff --git a/vulnerabilities/management/commands/purl2cpe.py b/vulnerabilities/management/commands/purl2cpe.py index 277348615..dc614242c 100644 --- a/vulnerabilities/management/commands/purl2cpe.py +++ b/vulnerabilities/management/commands/purl2cpe.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/middleware/ban_user_agent.py b/vulnerabilities/middleware/ban_user_agent.py index 6aafc490c..34332194d 100644 --- a/vulnerabilities/middleware/ban_user_agent.py +++ b/vulnerabilities/middleware/ban_user_agent.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py b/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py index 61cfc0531..17a86f58e 100644 --- a/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py +++ b/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py b/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py index d2c44c280..7c3efe166 100644 --- a/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py +++ b/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0061_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0061_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..a212d821c --- /dev/null +++ b/vulnerabilities/migrations/0061_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.13 on 2024-08-23 11:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0060_alter_kev_known_ransomware_campaign_use_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.0.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.0.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0062_package_is_ghost.py b/vulnerabilities/migrations/0062_package_is_ghost.py new file mode 100644 index 000000000..d64719045 --- /dev/null +++ b/vulnerabilities/migrations/0062_package_is_ghost.py @@ -0,0 +1,21 @@ +# Generated by Django 4.1.13 on 2024-08-23 12:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0061_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="package", + name="is_ghost", + field=models.BooleanField( + default=False, + help_text="True if the package does not exist in the upstream package manager or its repository.", + ), + ), + ] diff --git a/vulnerabilities/migrations/0063_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0063_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..a97d3024b --- /dev/null +++ b/vulnerabilities/migrations/0063_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2 on 2024-09-17 16:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0062_package_is_ghost"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.0.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.0.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py b/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py new file mode 100644 index 000000000..c4d1e83f3 --- /dev/null +++ b/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.15 on 2024-09-12 12:56 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline + from vulnerabilities.pipelines.pypa_importer import PyPaImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.npm.NpmImporter").update( + created_by=NpmImporterPipeline.pipeline_id + ) + Advisory.objects.filter(created_by="vulnerabilities.importers.pypa.PyPaImporter").update( + created_by=PyPaImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline + from vulnerabilities.pipelines.pypa_importer import PyPaImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=NpmImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.npm.NpmImporter" + ) + Advisory.objects.filter(created_by=PyPaImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.pypa.PyPaImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0063_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py new file mode 100644 index 000000000..80b43a954 --- /dev/null +++ b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-23 13:06 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").update( + created_by=NginxImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.nginx.NginxImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0064_update_npm_pypa_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py new file mode 100644 index 000000000..e72b0616b --- /dev/null +++ b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 13:08 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").update( + created_by=GitLabImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=GitLabImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0065_update_nginx_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0067_update_github_advisory_created_by.py b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py new file mode 100644 index 000000000..4b9bb8485 --- /dev/null +++ b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 14:31 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").update( + created_by=GitHubAPIImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=GitHubAPIImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.github.GitHubAPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0066_update_gitlab_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py new file mode 100644 index 000000000..2a91f55ee --- /dev/null +++ b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.15 on 2024-09-27 19:38 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").update( + created_by=NVDImporterPipeline.pipeline_id + ) + + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=NVDImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.nvd.NVDImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0067_update_github_advisory_created_by"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0069_exploit_delete_kev.py b/vulnerabilities/migrations/0069_exploit_delete_kev.py new file mode 100644 index 000000000..5c06911eb --- /dev/null +++ b/vulnerabilities/migrations/0069_exploit_delete_kev.py @@ -0,0 +1,131 @@ +# Generated by Django 4.2.15 on 2024-09-21 15:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0068_update_nvd_advisory_created_by"), + ] + + operations = [ + migrations.CreateModel( + name="Exploit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "date_added", + models.DateField( + blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", + null=True, + ), + ), + ( + "description", + models.TextField( + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", + null=True, + ), + ), + ( + "required_action", + models.TextField( + blank=True, + help_text="The required action to address the vulnerability, typically to apply vendor updates or apply vendor mitigations or to discontinue use.", + null=True, + ), + ), + ( + "due_date", + models.DateField( + blank=True, + help_text="The date the required action is due, which applies to all USA federal civilian executive branch (FCEB) agencies, but all organizations are strongly encouraged to execute the required action", + null=True, + ), + ), + ( + "notes", + models.TextField( + blank=True, + help_text="Additional notes and resources about the vulnerability, often a URL to vendor instructions.", + null=True, + ), + ), + ( + "known_ransomware_campaign_use", + models.BooleanField( + default=False, + help_text="Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; \n or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.", + ), + ), + ( + "source_date_published", + models.DateField( + blank=True, + help_text="The date that the exploit was published or disclosed.", + null=True, + ), + ), + ( + "exploit_type", + models.TextField( + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "platform", + models.TextField( + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + null=True, + ), + ), + ( + "source_date_updated", + models.DateField( + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + null=True, + ), + ), + ( + "data_source", + models.TextField( + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + null=True, + ), + ), + ( + "source_url", + models.URLField( + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", + null=True, + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="exploits", + to="vulnerabilities.vulnerability", + ), + ), + ], + ), + migrations.DeleteModel( + name="Kev", + ), + ] diff --git a/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py new file mode 100644 index 000000000..ccec92bde --- /dev/null +++ b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.15 on 2024-10-07 12:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0069_exploit_delete_kev"), + ] + + operations = [ + migrations.AlterField( + model_name="advisory", + name="created_by", + field=models.CharField( + help_text="Fully qualified name of the importer prefixed with themodule name importing the advisory. Eg:vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", + max_length=100, + ), + ), + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.0.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.0.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] \ No newline at end of file diff --git a/vulnerabilities/migrations/0071_auto_20241007_1044.py b/vulnerabilities/migrations/0071_auto_20241007_1044.py new file mode 100644 index 000000000..182c2f532 --- /dev/null +++ b/vulnerabilities/migrations/0071_auto_20241007_1044.py @@ -0,0 +1,197 @@ +from django.db import migrations, models +import django.db.models.deletion +from aboutcode.pipeline import LoopProgress + +def split_packagerelatedvulnerability(apps, schema_editor): + PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability') + FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability') + AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability') + + obsolete_package_relation_query = PackageRelatedVulnerability.objects.all() + obsolete_package_relation_query_count = obsolete_package_relation_query.count() + print(f"\nMigrating {obsolete_package_relation_query_count:,d} old package vulnerability relationship.") + + progress = LoopProgress( + total_iterations=obsolete_package_relation_query_count, + progress_step=1, + logger=print, + ) + fixing_packages = [] + affected_packages = [] + for prv in progress.iter(obsolete_package_relation_query.iterator(chunk_size=10000)): + if prv.fix: + fp = FixingPackageRelatedVulnerability( + package=prv.package, + vulnerability=prv.vulnerability, + created_by=prv.created_by, + confidence=prv.confidence, + ) + fixing_packages.append(fp) + else: + ap = AffectedByPackageRelatedVulnerability( + package=prv.package, + vulnerability=prv.vulnerability, + created_by=prv.created_by, + confidence=prv.confidence, + ) + affected_packages.append(ap) + + FixingPackageRelatedVulnerability.objects.bulk_create(fixing_packages, batch_size=10000) + AffectedByPackageRelatedVulnerability.objects.bulk_create(affected_packages, batch_size=10000) + +def reverse_migration(apps, schema_editor): + FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability') + AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability') + PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability') + + fixing_package_relation_query = FixingPackageRelatedVulnerability.objects.all() + fixing_package_relation_query_count = fixing_package_relation_query.count() + print(f"\nMigrating {fixing_package_relation_query_count:,d} FixingPackage to old relationship.") + + progress = LoopProgress( + total_iterations=fixing_package_relation_query_count, + progress_step=1, + logger=print, + ) + for fpv in progress.iter(fixing_package_relation_query.iterator(chunk_size=10000)): + PackageRelatedVulnerability.objects.create( + package=fpv.package, + vulnerability=fpv.vulnerability, + created_by=fpv.created_by, + confidence=fpv.confidence, + fix=True, + ) + + affected_package_relation_query = AffectedByPackageRelatedVulnerability.objects.all() + affected_package_relation_query_count = affected_package_relation_query.count() + print(f"\nMigrating {affected_package_relation_query_count:,d} AffectedPackage to old relationship.") + + progress = LoopProgress( + total_iterations=affected_package_relation_query_count, + progress_step=1, + logger=print, + ) + for apv in progress.iter(affected_package_relation_query.iterator(chunk_size=10000)): + PackageRelatedVulnerability.objects.create( + package=apv.package, + vulnerability=apv.vulnerability, + created_by=apv.created_by, + confidence=apv.confidence, + fix=False, + ) + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0070_alter_advisory_created_by_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="advisory", + name="created_by", + field=models.CharField( + help_text="Fully qualified name of the importer prefixed with themodule name importing the advisory. Eg:vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", + max_length=100, + ), + ), + migrations.CreateModel( + name="FixingPackageRelatedVulnerability", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created_by", + models.CharField( + blank=True, + help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover", + max_length=100, + ), + ), + ( + "confidence", + models.PositiveIntegerField( + default=100, + help_text="Confidence score for this relation", + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(100), + ], + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package" + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.vulnerability", + ), + ), + ], + options={ + "verbose_name_plural": "Fixing Package Related Vulnerabilities", + "ordering": ["package", "vulnerability"], + "abstract": False, + "unique_together": {("package", "vulnerability")}, + }, + ), + migrations.CreateModel( + name="AffectedByPackageRelatedVulnerability", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created_by", + models.CharField( + blank=True, + help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover", + max_length=100, + ), + ), + ( + "confidence", + models.PositiveIntegerField( + default=100, + help_text="Confidence score for this relation", + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(100), + ], + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package" + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.vulnerability", + ), + ), + ], + options={ + "verbose_name_plural": "Affected By Package Related Vulnerabilities", + "ordering": ["package", "vulnerability"], + "abstract": False, + "unique_together": {("package", "vulnerability")}, + }, + ), + migrations.RunPython(split_packagerelatedvulnerability, reverse_migration), + ] diff --git a/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py b/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py new file mode 100644 index 000000000..cf15473af --- /dev/null +++ b/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.15 on 2024-10-07 12:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0071_auto_20241007_1044"), + ] + + operations = [ + migrations.RemoveField( + model_name="package", + name="vulnerabilities", + ), + migrations.RemoveField( + model_name="vulnerability", + name="packages", + ), + migrations.AddField( + model_name="package", + name="affected_by_vulnerabilities", + field=models.ManyToManyField( + through="vulnerabilities.AffectedByPackageRelatedVulnerability", + to="vulnerabilities.vulnerability", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="affecting_packages", + field=models.ManyToManyField( + through="vulnerabilities.AffectedByPackageRelatedVulnerability", + to="vulnerabilities.package", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="fixed_by_packages", + field=models.ManyToManyField( + related_name="fixing_vulnerabilities", + through="vulnerabilities.FixingPackageRelatedVulnerability", + to="vulnerabilities.package", + ), + ), + ] diff --git a/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py b/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py new file mode 100644 index 000000000..a99147a16 --- /dev/null +++ b/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.15 on 2024-10-15 10:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0072_remove_package_vulnerabilities_and_more"), + ] + + operations = [ + migrations.DeleteModel( + name="PackageRelatedVulnerability", + ), + ] diff --git a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py new file mode 100644 index 000000000..d0e73181a --- /dev/null +++ b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.16 on 2024-10-24 13:51 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").update( + created_by=PyPIImporterPipeline.pipeline_id + ) + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=PyPIImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.pysec.PyPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0073_delete_packagerelatedvulnerability"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0075_package_risk_score.py b/vulnerabilities/migrations/0075_package_risk_score.py new file mode 100644 index 000000000..72827ae63 --- /dev/null +++ b/vulnerabilities/migrations/0075_package_risk_score.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.16 on 2024-10-29 10:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0074_update_pysec_advisory_created_by"), + ] + + operations = [ + migrations.AddField( + model_name="package", + name="risk_score", + field=models.DecimalField( + decimal_places=2, + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=4, + null=True, + ), + ), + ] diff --git a/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..6ff76eb90 --- /dev/null +++ b/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-08 07:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0075_package_risk_score"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.1.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.1.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..82862b3da --- /dev/null +++ b/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-11 12:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0076_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.2.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.2.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py b/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py new file mode 100644 index 000000000..c3e4aad38 --- /dev/null +++ b/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py @@ -0,0 +1,220 @@ +# Generated by Django 4.2.16 on 2024-11-01 12:40 + +from django.db import migrations, models +from aboutcode.pipeline import LoopProgress +from datetime import datetime +from datetime import timezone + + +class Migration(migrations.Migration): + """ + Add url field to the VulnerabilitySeverity model and remove reference foreignkey + relationship. Also, add new M2M 'severities' field in Vulnerability and + AffectedByPackageRelatedVulnerability relationship. + + To achieve this following model changes and data migrations are applied in chronological order: + - Make VulnerabilitySeverity reference field nullable (to make the migration reversible). + - Add 'severities' field to AffectedByPackageRelatedVulnerability. + - Add 'severities' field to Vulnerability. + - Add 'url' field to VulnerabilitySeverity. + - Data migration to remove corrupted SUSE scores. + - Data migration to enable reprocessing of old suse Advisory on next import. + - Data migration to populate new VulnerabilitySeverity url field using reference. + - Data migration to populate Vulnerability 'severities' M2M relationship. + - Delete VulnerabilitySeverity reference field. + """ + + def remove_inaccurate_suse_score(apps, schema_editor): + """ + Remove inaccurate suse severity scores. + See https://github.com/aboutcode-org/vulnerablecode/issues/1597 + """ + print(f"\nRemoving inaccurate suse severity scores.") + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + VulnerabilitySeverity.objects.filter( + reference__url="https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" + ).delete() + + def reverse_remove_inaccurate_suse_score(apps, schema_editor): + """Reverse data migration not needed for inaccurate severity scores.""" + pass + + def reprocess_suse_advisory_on_next_import(apps, schema_editor): + """Clear `date_imported` on old suse advisory to enable reprocessing on next suse import.""" + + print(f"\nEnable reprocess of old suse advisory on next import.") + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter( + created_by="vulnerabilities.importers.suse_scores.SUSESeverityScoreImporter" + ).update(date_imported=None) + + def reverse_reprocess_suse_advisory_on_next_import(apps, schema_editor): + """Populate `date_imported` on old suse advisory to prevent reprocessing on next suse import.""" + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter( + created_by="vulnerabilities.importers.suse_scores.SUSESeverityScoreImporter" + ).update(date_imported=datetime.now(timezone.utc)) + + def populate_severity_model_with_url(apps, schema_editor): + """Populate the new VulnerabilitySeverity `url` field using reference url""" + + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + chunk_size = 10000 + batch = [] + vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference") + + print( + f"\nPopulating {vulnerability_severity_query.count():,d} VulnerabilitySeverity" + "`url` field using reference url." + ) + progress = LoopProgress( + total_iterations=vulnerability_severity_query.count(), + progress_step=10, + logger=print, + ) + for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)): + severity.url = severity.reference.url + batch.append(severity) + + if len(batch) >= chunk_size: + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["url"], + ) + batch.clear() + + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["url"], + ) + + def reverse_populate_severity_model_reference_using_url(apps, schema_editor): + """Reverse: Populate the reference using `url` to ensure proper reverse migration.""" + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") + chunk_size = 10000 + batch = [] + vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference") + + print( + f"\nReverse: Populating {vulnerability_severity_query.count():,d} VulnerabilitySeverity" + "`reference` relation using url." + ) + progress = LoopProgress( + total_iterations=vulnerability_severity_query.count(), + progress_step=10, + logger=print, + ) + for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)): + severity.reference = VulnerabilityReference.objects.get(url=severity.url) + batch.append(severity) + + if len(batch) >= chunk_size: + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["reference"], + ) + batch.clear() + + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["reference"], + ) + + def populate_vulnerability_model_with_severities(apps, schema_editor): + """Populate the new Vulnerability `severities` relation using referenced severity.""" + Vulnerability = apps.get_model("vulnerabilities", "Vulnerability") + chunk_size = 10000 + + vulnerability_query = Vulnerability.objects.prefetch_related( + "references__vulnerabilityseverity_set" + ) + print( + f"\nPopulating {vulnerability_query.count():,d} Vulnerability `severities`" + "relation using referenced severity." + ) + progress = LoopProgress( + total_iterations=vulnerability_query.count(), + progress_step=10, + logger=print, + ) + for vulnerability in progress.iter(vulnerability_query.iterator(chunk_size=chunk_size)): + references = vulnerability.references.all() + severity_ids = [ + severity.id + for reference in references + for severity in reference.vulnerabilityseverity_set.all() + if reference.vulnerabilityseverity_set.exists() + ] + vulnerability.severities.set(severity_ids) + + def reverse_populate_vulnerability_model_with_severities(apps, schema_editor): + """Reverse data migration not needed for new `severities` relationship.""" + pass + + dependencies = [ + ("vulnerabilities", "0077_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + # reference field should nullable to properly support reverse migration. + migrations.AlterField( + model_name="vulnerabilityseverity", + name="reference", + field=models.ForeignKey( + to="vulnerabilities.vulnerabilityreference", + on_delete=models.CASCADE, + null=True, + ), + ), + migrations.AlterModelOptions( + name="vulnerabilityseverity", + options={"ordering": ["url", "scoring_system", "value"]}, + ), + migrations.AlterUniqueTogether( + name="vulnerabilityseverity", + unique_together=set(), + ), + migrations.AddField( + model_name="affectedbypackagerelatedvulnerability", + name="severities", + field=models.ManyToManyField( + related_name="affected_package_vulnerability_relations", + to="vulnerabilities.vulnerabilityseverity", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="severities", + field=models.ManyToManyField( + related_name="vulnerabilities", to="vulnerabilities.vulnerabilityseverity" + ), + ), + migrations.AddField( + model_name="vulnerabilityseverity", + name="url", + field=models.URLField( + help_text="URL to the vulnerability severity", max_length=1024, null=True + ), + ), + migrations.RunPython( + code=remove_inaccurate_suse_score, + reverse_code=reverse_remove_inaccurate_suse_score, + ), + migrations.RunPython( + code=reprocess_suse_advisory_on_next_import, + reverse_code=reverse_reprocess_suse_advisory_on_next_import, + ), + migrations.RunPython( + code=populate_severity_model_with_url, + reverse_code=reverse_populate_severity_model_reference_using_url, + ), + migrations.RunPython( + code=populate_vulnerability_model_with_severities, + reverse_code=reverse_populate_vulnerability_model_with_severities, + ), + migrations.RemoveField( + model_name="vulnerabilityseverity", + name="reference", + ), + ] diff --git a/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..a9f92c8af --- /dev/null +++ b/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-14 08:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0078_alter_vulnerabilityseverity_options_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..f535e7fe4 --- /dev/null +++ b/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-15 02:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0079_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..97126051a --- /dev/null +++ b/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-15 06:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0080_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py new file mode 100644 index 000000000..26a55e714 --- /dev/null +++ b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.16 on 2024-11-17 13:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0081_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="vulnerability", + name="exploitability", + field=models.DecimalField( + decimal_places=1, + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, applications, or networks. This metric is determined automatically based on the discovery of known exploits.", + max_digits=2, + null=True, + ), + ), + migrations.AddField( + model_name="vulnerability", + name="weighted_severity", + field=models.DecimalField( + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", + max_digits=3, + null=True, + ), + ), + migrations.AlterField( + model_name="package", + name="risk_score", + field=models.DecimalField( + decimal_places=1, + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=3, + null=True, + ), + ), + ] diff --git a/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..54c5a7b14 --- /dev/null +++ b/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.16 on 2024-11-15 11:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0082_vulnerability_exploitability_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + help_text="Version of the software at the time of change", max_length=100 + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + help_text="Version of the software at the time of change", max_length=100 + ), + ), + ] diff --git a/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py b/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py new file mode 100644 index 000000000..6b33c1a59 --- /dev/null +++ b/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.16 on 2024-12-04 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0083_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="package", + options={ + "ordering": [ + "type", + "namespace", + "name", + "version_rank", + "version", + "qualifiers", + "subpath", + ] + }, + ), + migrations.AddField( + model_name="package", + name="version_rank", + field=models.IntegerField( + default=0, + help_text="Rank of the version to support ordering by version. Rank zero means the rank has not been defined yet", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 26a856d8e..c2e89022f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3,13 +3,14 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import hashlib import json import logging +import typing from contextlib import suppress from functools import cached_property from typing import Optional @@ -42,6 +43,7 @@ from univers.version_range import AlpineLinuxVersionRange from univers.versions import Version +from aboutcode import hashid from vulnerabilities import utils from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import normalize_purl @@ -85,7 +87,7 @@ def affecting_vulnerabilities(self): """ Return a queryset of Vulnerability that affect a package. """ - return self.filter(packagerelatedvulnerability__fix=False) + return self.filter(affecting_packages__isnull=False) def with_cpes(self): """ @@ -149,15 +151,47 @@ def search(self, query: str = None): def with_package_counts(self): return self.annotate( - vulnerable_package_count=Count( - "packages", filter=Q(packagerelatedvulnerability__fix=False), distinct=True - ), - patched_package_count=Count( - "packages", filter=Q(packagerelatedvulnerability__fix=True), distinct=True - ), + vulnerable_package_count=Count("affecting_packages", distinct=True), + patched_package_count=Count("fixed_by_packages", distinct=True), ) +class VulnerabilitySeverity(models.Model): + url = models.URLField( + max_length=1024, + null=True, + help_text="URL to the vulnerability severity", + ) + + scoring_system_choices = tuple( + (system.identifier, system.name) for system in SCORING_SYSTEMS.values() + ) + + scoring_system = models.CharField( + max_length=50, + choices=scoring_system_choices, + help_text="Identifier for the scoring system used. Available choices are: {} ".format( + ",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices) + ), + ) + + value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High") + + scoring_elements = models.CharField( + max_length=150, + null=True, + help_text="Supporting scoring elements used to compute the score values. " + "For example a CVSS vector string as used to compute a CVSS score.", + ) + + published_at = models.DateTimeField( + blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" + ) + + class Meta: + ordering = ["url", "scoring_system", "value"] + + class VulnerabilityStatusType(models.IntegerChoices): """List of vulnerability statuses.""" @@ -189,15 +223,53 @@ class Vulnerability(models.Model): to="VulnerabilityReference", through="VulnerabilityRelatedReference" ) - packages = models.ManyToManyField( + affecting_packages = models.ManyToManyField( + to="Package", + through="AffectedByPackageRelatedVulnerability", + ) + + fixed_by_packages = models.ManyToManyField( to="Package", - through="PackageRelatedVulnerability", + through="FixingPackageRelatedVulnerability", + related_name="fixing_vulnerabilities", # Unique related_name ) status = models.IntegerField( choices=VulnerabilityStatusType.choices, default=VulnerabilityStatusType.PUBLISHED ) + severities = models.ManyToManyField( + VulnerabilitySeverity, + related_name="vulnerabilities", + ) + + exploitability = models.DecimalField( + null=True, + max_digits=2, + decimal_places=1, + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, " + "applications, or networks. This metric is determined automatically based on the discovery of known exploits.", + ) + + weighted_severity = models.DecimalField( + null=True, + max_digits=3, + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", + ) + + @property + def risk_score(self): + """ + Risk expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + Risk = min(weighted severity * exploitability, 10) + """ + if self.exploitability and self.weighted_severity: + risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) + return round(risk_score, 1) + objects = VulnerabilityQuerySet.as_manager() class Meta: @@ -211,32 +283,25 @@ def __str__(self): def vcid(self): return self.vulnerability_id - @property - def severities(self): - """ - Return a queryset of VulnerabilitySeverity for this vulnerability. - """ - return VulnerabilitySeverity.objects.filter(reference__in=self.references.all()) - @property def affected_packages(self): """ Return a queryset of packages that are affected by this vulnerability. """ - return self.packages.affected() - - # legacy aliases - vulnerable_packages = affected_packages + return self.affecting_packages.with_is_vulnerable() @property - def fixed_by_packages(self): + def packages_fixing(self): """ Return a queryset of packages that are fixing this vulnerability. """ - return self.packages.fixing() + return self.fixed_by_packages + + # legacy aliases + vulnerable_packages = affected_packages # legacy alias - patched_packages = fixed_by_packages + patched_packages = packages_fixing @property def get_aliases(self): @@ -266,9 +331,9 @@ def get_details_url(self, request): """ Return this Package details URL. """ - from rest_framework.reverse import reverse + from rest_framework.reverse import reverse as reved - return reverse( + return reved( "vulnerability_details", kwargs={"vulnerability_id": self.vulnerability_id}, request=request, @@ -314,6 +379,10 @@ class Weakness(models.Model): vulnerabilities = models.ManyToManyField(Vulnerability, related_name="weaknesses") db = Database() + @property + def cwe(self): + return f"CWE-{self.cwe_id}" + @property def weakness(self): """ @@ -398,7 +467,7 @@ def __str__(self): @property def is_cpe(self): """ - Return Trueis this is a CPE reference. + Return True if this is a CPE reference. """ return self.reference_id.startswith("cpe") @@ -438,7 +507,7 @@ def get_fixed_by_package_versions(self, purl: PackageURL, fix=True): } if fix: - filter_dict["packagerelatedvulnerability__fix"] = True + filter_dict["fixing_vulnerabilities__isnull"] = False return Package.objects.filter(**filter_dict).distinct() @@ -460,7 +529,7 @@ def affected(self): """ Return only packages affected by a vulnerability. """ - return self.filter(packagerelatedvulnerability__fix=False) + return self.filter(affected_by_vulnerabilities__isnull=False) vulnerable = affected @@ -468,17 +537,15 @@ def fixing(self): """ Return only packages fixing a vulnerability . """ - return self.filter(packagerelatedvulnerability__fix=True) + return self.filter(fixing_vulnerabilities__isnull=False) def with_vulnerability_counts(self): return self.annotate( vulnerability_count=Count( - "vulnerabilities", - filter=Q(packagerelatedvulnerability__fix=False), + "affected_by_vulnerabilities", ), patched_vulnerability_count=Count( - "vulnerabilities", - filter=Q(packagerelatedvulnerability__fix=True), + "fixing_vulnerabilities", ), ) @@ -517,7 +584,13 @@ def for_purl(self, purl): """ Return a queryset matching the ``purl`` Package URL. """ - return self.filter(**purl_to_dict(purl)) + return self.filter(package_url=purl) + + def for_purls(self, purls=()): + """ + Return a queryset of Packages matching a list of PURLs. + """ + return self.filter(package_url__in=purls).distinct() def with_cpes(self): """ @@ -527,34 +600,30 @@ def with_cpes(self): def for_cpe(self, cpe): """ - Return a queryset of Vulnerability that have the ``cpe`` as an NVD CPE reference. + Return a queryset of Packages that have the ``cpe`` as an NVD CPE reference. """ return self.filter(vulnerabilities__vulnerabilityreference__reference_id__exact=cpe) def with_cves(self): """ - Return a queryset of Vulnerability that have one or more NVD CVE aliases. + Return a queryset of Packages that have one or more NVD CVE aliases. """ return self.filter(vulnerabilities__aliases__alias__startswith="CVE") def for_cve(self, cve): """ - Return a queryset of Vulnerability that have the the NVD CVE ``cve`` as an alias. + Return a queryset of Packages that have the NVD CVE ``cve`` as a vulnerability alias. """ - return self.filter(vulnerabilities__vulnerabilityreference__reference_id__exact=cve) - - def for_purls(self, purls=[]): - return Package.objects.filter(package_url__in=purls).distinct() + return self.filter(vulnerabilities__aliases__alias=cve) def with_is_vulnerable(self): """ - Annotate Package with ``with_is_vulnerable`` boolean attribute. + Annotate Package with ``is_vulnerable`` boolean attribute. """ return self.annotate( is_vulnerable=Exists( - PackageRelatedVulnerability.objects.filter( + AffectedByPackageRelatedVulnerability.objects.filter( package=OuterRef("pk"), - fix=False, ) ) ) @@ -563,7 +632,7 @@ def only_vulnerable(self): return self._vulnerable(True) def only_non_vulnerable(self): - return self._vulnerable(False) + return self._vulnerable(False).filter(is_ghost=False) def _vulnerable(self, vulnerable=True): """ @@ -571,6 +640,12 @@ def _vulnerable(self, vulnerable=True): """ return self.with_is_vulnerable().filter(is_vulnerable=vulnerable) + def vulnerable(self): + """ + Return only packages that are vulnerable. + """ + return self.filter(affected_by_vulnerabilities__isnull=False) + def get_purl_query_lookups(purl): """ @@ -592,8 +667,15 @@ class Package(PackageURLMixin): # https://github.com/package-url/packageurl-python/pull/67 # gets merged - vulnerabilities = models.ManyToManyField( - to="Vulnerability", through="PackageRelatedVulnerability" + affected_by_vulnerabilities = models.ManyToManyField( + to="Vulnerability", + through="AffectedByPackageRelatedVulnerability", + ) + + fixing_vulnerabilities = models.ManyToManyField( + to="Vulnerability", + through="FixingPackageRelatedVulnerability", + related_name="fixed_by_packages", # Unique related_name ) package_url = models.CharField( @@ -610,6 +692,25 @@ class Package(PackageURLMixin): db_index=True, ) + is_ghost = models.BooleanField( + default=False, + help_text="True if the package does not exist in the upstream package manager or its repository.", + ) + + risk_score = models.DecimalField( + null=True, + max_digits=3, + decimal_places=1, + help_text="Risk score between 0.00 and 10.00, where higher values " + "indicate greater vulnerability risk for the package.", + ) + + version_rank = models.IntegerField( + help_text="Rank of the version to support ordering by version. Rank " + "zero means the rank has not been defined yet", + default=0, + ) + objects = PackageQuerySet.as_manager() def save(self, *args, **kwargs): @@ -643,17 +744,40 @@ def purl(self): class Meta: unique_together = ["type", "namespace", "name", "version", "qualifiers", "subpath"] - ordering = ["type", "namespace", "name", "version", "qualifiers", "subpath"] + ordering = ["type", "namespace", "name", "version_rank", "version", "qualifiers", "subpath"] def __str__(self): return self.package_url + @property + def calculate_version_rank(self): + """ + Calculate and return the `version_rank` for a package that does not have one. + If this package already has a `version_rank`, return it. + + The calculated rank will be interpolated between two packages that have + `version_rank` values and are closest to this package in terms of version order. + """ + + group_packages = Package.objects.filter( + type=self.type, + namespace=self.namespace, + name=self.name, + ) + + if any(p.version_rank == 0 for p in group_packages): + sorted_packages = sorted(group_packages, key=lambda p: self.version_class(p.version)) + for rank, package in enumerate(sorted_packages, start=1): + package.version_rank = rank + Package.objects.bulk_update(sorted_packages, fields=["version_rank"]) + return self.version_rank + @property def affected_by(self): """ Return a queryset of vulnerabilities affecting this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + return self.affected_by_vulnerabilities.all() # legacy aliases vulnerable_to = affected_by @@ -664,7 +788,7 @@ def fixing(self): """ Return a queryset of vulnerabilities fixed by this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=True) + return self.fixing_vulnerabilities.all() # legacy aliases resolved_to = fixing @@ -694,14 +818,6 @@ def get_details_url(self, request): return reverse("package_details", kwargs={"purl": self.purl}, request=request) - def sort_by_version(self, packages): - """ - Return a sequence of `packages` sorted by version. - """ - if not packages: - return [] - return sorted(packages, key=lambda x: self.version_class(x.version)) - @cached_property def version_class(self): range_class = RANGE_CLASS_BY_SCHEMES.get(self.type) @@ -712,61 +828,81 @@ def current_version(self): return self.version_class(self.version) @property - def fixed_package_details(self): + def next_non_vulnerable_version(self): """ - Return a mapping of vulnerabilities that affect this package and the next and - latest non-vulnerable versions. + Return the version string of the next non-vulnerable package version. """ - package_details = {} - package_details["purl"] = PackageURL.from_string(self.purl) + next_non_vulnerable, _ = self.get_non_vulnerable_versions() + return next_non_vulnerable.version if next_non_vulnerable else None - next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() - package_details["next_non_vulnerable"] = next_non_vulnerable - package_details["latest_non_vulnerable"] = latest_non_vulnerable - - package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() + @property + def vulnerabilities(self): + return self.affected_by_vulnerabilities.all() | self.fixing_vulnerabilities.all() - return package_details + @property + def latest_non_vulnerable_version(self): + """ + Return the version string of the latest non-vulnerable package version. + """ + _, latest_non_vulnerable = self.get_non_vulnerable_versions() + return latest_non_vulnerable.version if latest_non_vulnerable else None def get_non_vulnerable_versions(self): """ - Return a tuple of the next and latest non-vulnerable versions as PackageURLs. Return a tuple of - (None, None) if there is no non-vulnerable version. + Return a tuple of the next and latest non-vulnerable versions as Package instance. + Return a tuple of (None, None) if there is no non-vulnerable version. """ + if self.version_rank == 0: + self.calculate_version_rank non_vulnerable_versions = Package.objects.get_fixed_by_package_versions( self, fix=False ).only_non_vulnerable() - sorted_versions = self.sort_by_version(non_vulnerable_versions) - - later_non_vulnerable_versions = [] - for non_vuln_ver in sorted_versions: - if self.version_class(non_vuln_ver.version) > self.current_version: - later_non_vulnerable_versions.append(non_vuln_ver) - if later_non_vulnerable_versions: - sorted_versions = self.sort_by_version(later_non_vulnerable_versions) - next_non_vulnerable_version = sorted_versions[0] - latest_non_vulnerable_version = sorted_versions[-1] + later_non_vulnerable_versions = non_vulnerable_versions.filter( + version_rank__gt=self.version_rank + ) - next_non_vulnerable = PackageURL.from_string(next_non_vulnerable_version.purl) - latest_non_vulnerable = PackageURL.from_string(latest_non_vulnerable_version.purl) + later_non_vulnerable_versions = list(later_non_vulnerable_versions) + if later_non_vulnerable_versions: + sorted_versions = later_non_vulnerable_versions + next_non_vulnerable = sorted_versions[0] + latest_non_vulnerable = sorted_versions[-1] return next_non_vulnerable, latest_non_vulnerable return None, None + @property + def fixed_package_details(self): + """ + Return a mapping of vulnerabilities that affect this package and the next and + latest non-vulnerable versions. + """ + package_details = {} + package_details["purl"] = PackageURL.from_string(self.purl) + + next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() + package_details["next_non_vulnerable"] = next_non_vulnerable + package_details["latest_non_vulnerable"] = latest_non_vulnerable + + package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() + + return package_details + def get_affecting_vulnerabilities(self): """ Return a list of vulnerabilities that affect this package together with information regarding the versions that fix the vulnerabilities. """ + if self.version_rank == 0: + self.calculate_version_rank package_details_vulns = [] fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) - package_vulnerabilities = self.vulnerabilities.affecting_vulnerabilities().prefetch_related( + package_vulnerabilities = self.affected_by_vulnerabilities.prefetch_related( Prefetch( - "packages", + "fixed_by_packages", queryset=fixed_by_packages, to_attr="fixed_packages", ) @@ -783,12 +919,13 @@ def get_affecting_vulnerabilities(self): if fixed_version > self.current_version: later_fixed_packages.append(fixed_pkg) - next_fixed_package = None next_fixed_package_vulns = [] sort_fixed_by_packages_by_version = [] if later_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) + sort_fixed_by_packages_by_version = sorted( + later_fixed_packages, key=lambda p: p.version_rank + ) fixed_by_pkgs = [] @@ -816,40 +953,50 @@ def get_affecting_vulnerabilities(self): @property def fixing_vulnerabilities(self): """ - Return a queryset of Vulnerabilities that are fixed by this `package`. + Return a queryset of Vulnerabilities that are fixed by this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=True) + print("A") + return self.fixed_by_vulnerabilities.all() @property - def affecting_vulnerabilities(self): + def affecting_vulns(self): """ Return a queryset of Vulnerabilities that affect this `package`. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) + return self.affected_by_vulnerabilities.all().prefetch_related( + Prefetch( + "fixed_by_packages", + queryset=fixed_by_packages, + to_attr="fixed_packages", + ) + ) -class PackageRelatedVulnerability(models.Model): +class PackageRelatedVulnerabilityBase(models.Model): """ - Track the relationship between a Package and Vulnerability. + Abstract base class for package-vulnerability relations. """ - # TODO: Fix related_name package = models.ForeignKey( Package, on_delete=models.CASCADE, + # related_name="%(class)s_set", # Unique related_name per subclass ) vulnerability = models.ForeignKey( Vulnerability, on_delete=models.CASCADE, + # related_name="%(class)s_set", # Unique related_name per subclass ) created_by = models.CharField( max_length=100, blank=True, - help_text="Fully qualified name of the improver prefixed with the" - "module name responsible for creating this relation. Eg:" - "vulnerabilities.importers.nginx.NginxBasicImprover", + help_text=( + "Fully qualified name of the improver prefixed with the module name " + "responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover" + ), ) from vulnerabilities.improver import MAX_CONFIDENCE @@ -860,54 +1007,45 @@ class PackageRelatedVulnerability(models.Model): help_text="Confidence score for this relation", ) - fix = models.BooleanField( - default=False, - db_index=True, - help_text="Does this relation fix the specified vulnerability ?", - ) - class Meta: + abstract = True unique_together = ["package", "vulnerability"] - verbose_name_plural = "PackageRelatedVulnerabilities" - indexes = [models.Index(fields=["fix"])] ordering = ["package", "vulnerability"] def __str__(self): - return f"{self.package.package_url} {self.vulnerability.vulnerability_id}" + relation = "fixes" if isinstance(self, FixingPackageRelatedVulnerability) else "affected by" + return f"{self.package.package_url} {relation} {self.vulnerability.vulnerability_id}" def update_or_create(self, advisory): """ - Update if supplied record has more confidence than existing record - Create if doesn't exist + Update if supplied record has more confidence than existing record. + Create if it doesn't exist. """ + model_class = self.__class__ try: - existing = PackageRelatedVulnerability.objects.get( + existing = model_class.objects.get( vulnerability=self.vulnerability, package=self.package ) if self.confidence > existing.confidence: existing.created_by = self.created_by existing.confidence = self.confidence - existing.fix = self.fix existing.save() - # TODO: later we want these to be part of a log field in the DB logger.info( f"Confidence improved for {self.package} R {self.vulnerability}, " f"new confidence: {self.confidence}" ) self.add_package_vulnerability_changelog(advisory=advisory) - - except self.DoesNotExist: - PackageRelatedVulnerability.objects.create( + except model_class.DoesNotExist: + model_class.objects.create( vulnerability=self.vulnerability, created_by=self.created_by, package=self.package, confidence=self.confidence, - fix=self.fix, ) logger.info( f"New relationship {self.package} R {self.vulnerability}, " - f"fix: {self.fix}, confidence: {self.confidence}" + f"confidence: {self.confidence}" ) self.add_package_vulnerability_changelog(advisory=advisory) @@ -917,7 +1055,7 @@ def add_package_vulnerability_changelog(self, advisory): from vulnerabilities.utils import get_importer_name importer_name = get_importer_name(advisory) - if self.fix: + if isinstance(self, FixingPackageRelatedVulnerability): change_logger = PackageChangeLog.log_fixing else: change_logger = PackageChangeLog.log_affected_by @@ -929,37 +1067,20 @@ def add_package_vulnerability_changelog(self, advisory): ) -class VulnerabilitySeverity(models.Model): - reference = models.ForeignKey(VulnerabilityReference, on_delete=models.CASCADE) - - scoring_system_choices = tuple( - (system.identifier, system.name) for system in SCORING_SYSTEMS.values() - ) - - scoring_system = models.CharField( - max_length=50, - choices=scoring_system_choices, - help_text="Identifier for the scoring system used. Available choices are: {} ".format( - ",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices) - ), - ) +class FixingPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): + class Meta(PackageRelatedVulnerabilityBase.Meta): + verbose_name_plural = "Fixing Package Related Vulnerabilities" - value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High") - scoring_elements = models.CharField( - max_length=150, - null=True, - help_text="Supporting scoring elements used to compute the score values. " - "For example a CVSS vector string as used to compute a CVSS score.", - ) +class AffectedByPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): - published_at = models.DateTimeField( - blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" + severities = models.ManyToManyField( + VulnerabilitySeverity, + related_name="affected_package_vulnerability_relations", ) - class Meta: - unique_together = ["reference", "scoring_system", "value"] - ordering = ["reference", "scoring_system", "value"] + class Meta(PackageRelatedVulnerabilityBase.Meta): + verbose_name_plural = "Affected By Package Related Vulnerabilities" class AliasQuerySet(BaseQuerySet): @@ -1060,7 +1181,7 @@ class Advisory(models.Model): max_length=100, help_text="Fully qualified name of the importer prefixed with the" "module name importing the advisory. Eg:" - "vulnerabilities.importers.nginx.NginxImporter", + "vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", ) url = models.URLField( blank=True, @@ -1094,7 +1215,9 @@ def to_advisory_data(self) -> "AdvisoryData": return AdvisoryData( aliases=self.aliases, summary=self.summary, - affected_packages=[AffectedPackage.from_dict(pkg) for pkg in self.affected_packages], + affected_packages=[ + AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg + ], references=[Reference.from_dict(ref) for ref in self.references], date_published=self.date_published, weaknesses=self.weaknesses, @@ -1184,7 +1307,8 @@ class ChangeLog(models.Model): software_version = models.CharField( max_length=100, help_text="Version of the software at the time of change", - default=VULNERABLECODE_VERSION, + blank=False, + null=False, ) @property @@ -1346,49 +1470,90 @@ def log_fixing(cls, package, importer, source_url, related_vulnerability): ) -class Kev(models.Model): +class Exploit(models.Model): """ - Known Exploited Vulnerabilities + A vulnerability exploit is code used to + take advantage of a security flaw for unauthorized access or malicious activity. """ - vulnerability = models.OneToOneField( + vulnerability = models.ForeignKey( Vulnerability, + related_name="exploits", on_delete=models.CASCADE, - related_name="kev", ) date_added = models.DateField( - help_text="The date the vulnerability was added to the Known Exploited Vulnerabilities" - " (KEV) catalog in the format YYYY-MM-DD.", null=True, blank=True, + help_text="The date the vulnerability was added to an exploit catalog.", ) description = models.TextField( - help_text="Description of the vulnerability in the Known Exploited Vulnerabilities" - " (KEV) catalog, usually a refinement of the original CVE description" + null=True, + blank=True, + help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", ) required_action = models.TextField( + null=True, + blank=True, help_text="The required action to address the vulnerability, typically to " - "apply vendor updates or apply vendor mitigations or to discontinue use." + "apply vendor updates or apply vendor mitigations or to discontinue use.", ) due_date = models.DateField( - help_text="The date the required action is due in the format YYYY-MM-DD," - "which applies to all USA federal civilian executive branch (FCEB) agencies," - "but all organizations are strongly encouraged to execute the required action." + null=True, + blank=True, + help_text="The date the required action is due, which applies" + " to all USA federal civilian executive branch (FCEB) agencies, " + "but all organizations are strongly encouraged to execute the required action", ) - resources_and_notes = models.TextField( + notes = models.TextField( + null=True, + blank=True, help_text="Additional notes and resources about the vulnerability," - " often a URL to vendor instructions." + " often a URL to vendor instructions.", ) known_ransomware_campaign_use = models.BooleanField( default=False, - help_text="""Known if this vulnerability is known to have been leveraged as part of a ransomware campaign; - or 'Unknown' if CISA lacks confirmation that the vulnerability has been utilized for ransomware.""", + help_text="""Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; + or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.""", + ) + + source_date_published = models.DateField( + null=True, blank=True, help_text="The date that the exploit was published or disclosed." + ) + + exploit_type = models.TextField( + null=True, + blank=True, + help_text="The type of the exploit as provided by the original upstream data source.", + ) + + platform = models.TextField( + null=True, + blank=True, + help_text="The platform associated with the exploit as provided by the original upstream data source.", + ) + + source_date_updated = models.DateField( + null=True, + blank=True, + help_text="The date the exploit was updated in the original upstream data source.", + ) + + data_source = models.TextField( + null=True, + blank=True, + help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", + ) + + source_url = models.URLField( + null=True, + blank=True, + help_text="The URL to the exploit as provided in the original upstream data source.", ) @property diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py index 2a958312c..fd2b114a7 100755 --- a/vulnerabilities/oval_parser.py +++ b/vulnerabilities/oval_parser.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py new file mode 100644 index 000000000..d74db9f35 --- /dev/null +++ b/vulnerabilities/pipelines/__init__.py @@ -0,0 +1,192 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from datetime import datetime +from datetime import timezone +from timeit import default_timer as timer +from traceback import format_exc as traceback_format_exc +from typing import Iterable + +from aboutcode.pipeline import BasePipeline +from aboutcode.pipeline import LoopProgress +from aboutcode.pipeline import humanize_time + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.improver import MAX_CONFIDENCE +from vulnerabilities.models import Advisory +from vulnerabilities.pipes.advisory import import_advisory +from vulnerabilities.pipes.advisory import insert_advisory +from vulnerabilities.utils import classproperty + +module_logger = logging.getLogger(__name__) + + +class VulnerableCodePipeline(BasePipeline): + pipeline_id = None # Unique Pipeline ID + + def on_failure(self): + """ + Tasks to run in the event that pipeline execution fails. + + Implement cleanup or other tasks that need to be performed + on pipeline failure, such as: + - Removing cloned repositories. + - Deleting downloaded archives. + """ + pass + + def execute(self): + """Execute each steps in the order defined on this pipeline class.""" + self.log(f"Pipeline [{self.pipeline_name}] starting") + + steps = self.pipeline_class.get_steps(groups=self.selected_groups) + steps_count = len(steps) + pipeline_start_time = timer() + + for current_index, step in enumerate(steps, start=1): + step_name = step.__name__ + + if self.selected_steps and step_name not in self.selected_steps: + self.log(f"Step [{step_name}] skipped") + continue + + self.set_current_step(f"{current_index}/{steps_count} {step_name}") + self.log(f"Step [{step_name}] starting") + step_start_time = timer() + + try: + step(self) + except Exception as exception: + self.log("Pipeline failed") + on_failure_start_time = timer() + self.log(f"Running [on_failure] tasks") + self.on_failure() + on_failure_run_time = timer() - on_failure_start_time + self.log(f"Completed [on_failure] tasks in {humanize_time(on_failure_run_time)}") + + return 1, self.output_from_exception(exception) + + step_run_time = timer() - step_start_time + self.log(f"Step [{step_name}] completed in {humanize_time(step_run_time)}") + + self.set_current_step("") # Reset the `current_step` field on completion + pipeline_run_time = timer() - pipeline_start_time + self.log(f"Pipeline completed in {humanize_time(pipeline_run_time)}") + + return 0, "" + + def log(self, message, level=logging.INFO): + """Log the given `message` to the current module logger and execution_log.""" + now_local = datetime.now(timezone.utc).astimezone() + timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + message = f"{timestamp} {message}" + module_logger.log(level, message) + self.append_to_log(message) + + @classproperty + def pipeline_id(cls): + """Return unique pipeline_id set in cls.pipeline_id""" + + if cls.pipeline_id is None or cls.pipeline_id == "": + raise NotImplementedError("pipeline_id is not defined or is empty") + return cls.pipeline_id + + +class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline): + """ + Base importer pipeline for importing advisories. + + Uses: + Subclass this Pipeline and implement ``advisories_count`` and ``collect_advisories`` + method. Also override the ``steps`` and ``advisory_confidence`` as needed. + """ + + pipeline_id = None # Unique Pipeline ID, this should be the name of pipeline module. + license_url = None + spdx_license_expression = None + repo_url = None + importer_name = None + advisory_confidence = MAX_CONFIDENCE + + @classmethod + def steps(cls): + return ( + # Add step for downloading/cloning resource as required. + cls.collect_and_store_advisories, + cls.import_new_advisories, + # Add step for removing downloaded/cloned resource as required. + ) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + """ + Yield AdvisoryData for importer pipeline. + + Populate the `self.collected_advisories_count` field and yield AdvisoryData + """ + raise NotImplementedError + + def advisories_count(self) -> int: + """ + Return the estimated AdvisoryData to be yielded by ``collect_advisories``. + + Used by ``collect_and_store_advisories`` to log the progress of advisory collection. + """ + raise NotImplementedError + + def collect_and_store_advisories(self): + collected_advisory_count = 0 + estimated_advisory_count = self.advisories_count() + + if estimated_advisory_count > 0: + self.log(f"Collecting {estimated_advisory_count:,d} advisories") + + progress = LoopProgress(total_iterations=estimated_advisory_count, logger=self.log) + for advisory in progress.iter(self.collect_advisories()): + if _obj := insert_advisory( + advisory=advisory, + pipeline_id=self.pipeline_id, + logger=self.log, + ): + collected_advisory_count += 1 + + self.log(f"Successfully collected {collected_advisory_count:,d} advisories") + + def import_new_advisories(self): + new_advisories = Advisory.objects.filter( + created_by=self.pipeline_id, + date_imported__isnull=True, + ) + + new_advisories_count = new_advisories.count() + + self.log(f"Importing {new_advisories_count:,d} new advisories") + + imported_advisory_count = 0 + progress = LoopProgress(total_iterations=new_advisories_count, logger=self.log) + for advisory in progress.iter(new_advisories.paginated()): + self.import_advisory(advisory=advisory) + if advisory.date_imported: + imported_advisory_count += 1 + + self.log(f"Successfully imported {imported_advisory_count:,d} new advisories") + + def import_advisory(self, advisory: Advisory) -> int: + try: + import_advisory( + advisory=advisory, + pipeline_id=self.pipeline_id, + confidence=self.advisory_confidence, + logger=self.log, + ) + except Exception as e: + self.log( + f"Failed to import advisory: {advisory!r} with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py new file mode 100644 index 000000000..7ac4de838 --- /dev/null +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -0,0 +1,147 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from aboutcode.pipeline import LoopProgress +from django.db.models import Prefetch + +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.risk import compute_package_risk +from vulnerabilities.risk import compute_vulnerability_risk_factors + + +class ComputePackageRiskPipeline(VulnerableCodePipeline): + """ + Compute risk score for packages. + + See https://github.com/aboutcode-org/vulnerablecode/issues/1543 + """ + + pipeline_id = "compute_package_risk" + license_expression = None + + @classmethod + def steps(cls): + return ( + cls.compute_and_store_vulnerability_risk_score, + cls.compute_and_store_package_risk_score, + ) + + def compute_and_store_vulnerability_risk_score(self): + affected_vulnerabilities = ( + Vulnerability.objects.filter(affecting_packages__isnull=False) + .prefetch_related( + "references", + "severities", + "exploits", + ) + .distinct() + ) + + self.log( + f"Calculating risk for {affected_vulnerabilities.count():,d} vulnerability with a affected packages records" + ) + + progress = LoopProgress(total_iterations=affected_vulnerabilities.count(), logger=self.log) + + updatables = [] + updated_vulnerability_count = 0 + batch_size = 5000 + + for vulnerability in progress.iter(affected_vulnerabilities.paginated(per_page=batch_size)): + severities = vulnerability.severities.all() + references = vulnerability.references.all() + exploits = vulnerability.exploits.all() + + weighted_severity, exploitability = compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + vulnerability.weighted_severity = weighted_severity + vulnerability.exploitability = exploitability + + updatables.append(vulnerability) + + if len(updatables) >= batch_size: + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], + logger=self.log, + ) + + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], + logger=self.log, + ) + + self.log( + f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" + ) + + def compute_and_store_package_risk_score(self): + affected_packages = ( + Package.objects.filter(affected_by_vulnerabilities__isnull=False).prefetch_related( + Prefetch( + "affectedbypackagerelatedvulnerability_set__vulnerability", + queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), + ), + ) + ).distinct() + + self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") + + progress = LoopProgress( + total_iterations=affected_packages.count(), + logger=self.log, + progress_step=5, + ) + + updatables = [] + updated_package_count = 0 + batch_size = 10000 + + for package in progress.iter(affected_packages.paginated(per_page=batch_size)): + risk_score = compute_package_risk(package) + + if not risk_score: + continue + + package.risk_score = risk_score + updatables.append(package) + + if len(updatables) >= batch_size: + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], + logger=self.log, + ) + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], + logger=self.log, + ) + self.log(f"Successfully added risk score for {updated_package_count:,d} package") + + +def bulk_update(model, items, fields, logger): + item_count = 0 + if items: + try: + model.objects.bulk_update(objs=items, fields=fields) + item_count += len(items) + except Exception as e: + logger(f"Error updating {model.__name__}: {e}") + items.clear() + return item_count diff --git a/vulnerabilities/pipelines/compute_package_version_rank.py b/vulnerabilities/pipelines/compute_package_version_rank.py new file mode 100644 index 000000000..73d4aa60a --- /dev/null +++ b/vulnerabilities/pipelines/compute_package_version_rank.py @@ -0,0 +1,93 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from itertools import groupby + +from aboutcode.pipeline import LoopProgress +from django.db import transaction +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.versions import Version + +from vulnerabilities.models import Package +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ComputeVersionRankPipeline(VulnerableCodePipeline): + """ + A pipeline to compute and assign version ranks for all packages. + """ + + pipeline_id = "compute_version_rank" + license_expression = None + + @classmethod + def steps(cls): + return (cls.compute_and_store_version_rank,) + + def compute_and_store_version_rank(self): + """ + Compute and assign version ranks to all packages. + """ + groups = Package.objects.only("type", "namespace", "name").order_by( + "type", "namespace", "name" + ) + + def key(package): + return package.type, package.namespace, package.name + + groups = groupby(groups, key=key) + + groups = [(list(x), list(y)) for x, y in groups] + + total_groups = len(groups) + self.log(f"Calculating `version_rank` for {total_groups:,d} groups of packages.") + + progress = LoopProgress( + total_iterations=total_groups, + logger=self.log, + progress_step=5, + ) + + for group, packages in progress.iter(groups): + type, namespace, name = group + if type not in RANGE_CLASS_BY_SCHEMES: + continue + self.update_version_rank_for_group(packages) + + self.log("Successfully populated `version_rank` for all packages.") + + @transaction.atomic + def update_version_rank_for_group(self, packages): + """ + Update the `version_rank` for all packages in a specific group. + """ + + # Sort the packages by version + sorted_packages = self.sort_packages_by_version(packages) + + # Assign version ranks + updates = [] + for rank, package in enumerate(sorted_packages, start=1): + package.version_rank = rank + updates.append(package) + + # Bulk update to save the ranks + Package.objects.bulk_update(updates, fields=["version_rank"]) + + def sort_packages_by_version(self, packages): + """ + Sort packages by version using `version_class`. + """ + + if not packages: + return [] + version_class = RANGE_CLASS_BY_SCHEMES.get(packages[0].type).version_class + if not version_class: + version_class = Version + return sorted(packages, key=lambda p: version_class(p.version)) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py new file mode 100644 index 000000000..b31ac4134 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -0,0 +1,167 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import csv +import io +import logging +from traceback import format_exc as traceback_format_exc + +import requests +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser +from django.db import DataError + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ExploitDBImproverPipeline(VulnerableCodePipeline): + """ + ExploitDB Improver Pipeline: Fetch ExploitDB data, iterate over it to find the vulnerability with + the specified alias, and create or update the ref and ref-type accordingly. + """ + + pipeline_id = "enhance_with_exploitdb" + spdx_license_expression = "GPL-2.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploit, + ) + + def fetch_exploits(self): + exploit_db_url = ( + "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" + ) + self.log(f"Fetching {exploit_db_url}") + + try: + response = requests.get(exploit_db_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + + self.exploit_data = io.StringIO(response.text) + + def add_exploit(self): + + csvreader = csv.DictReader(self.exploit_data) + + raw_data = list(csvreader) + fetched_exploit_count = len(raw_data) + + vulnerability_exploit_count = 0 + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for row in progress.iter(raw_data): + vulnerability_exploit_count += add_vulnerability_exploit(row, self.log) + + self.log( + f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" + ) + + +def add_vulnerability_exploit(row, logger): + vulnerabilities = set() + + aliases = row["codes"].split(";") if row["codes"] else [] + + if not aliases: + return 0 + + for raw_alias in aliases: + try: + if alias := Alias.objects.get(alias=raw_alias): + vulnerabilities.add(alias.vulnerability) + except Alias.DoesNotExist: + continue + + if not vulnerabilities: + logger(f"No vulnerability found for aliases {aliases}") + return 0 + + date_added = parse_date(row["date_added"]) + source_date_published = parse_date(row["date_published"]) + source_date_updated = parse_date(row["date_updated"]) + + for vulnerability in vulnerabilities: + add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) + try: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Exploit-DB", + defaults={ + "date_added": date_added, + "description": row["description"], + "known_ransomware_campaign_use": row["verified"], + "source_date_published": source_date_published, + "exploit_type": row["type"], + "platform": row["platform"], + "source_date_updated": source_date_updated, + "source_url": row["source_url"], + }, + ) + except DataError as e: + logger( + f"Failed to Create the Vulnerability Exploit-DB with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + return 1 + + +def add_exploit_references(ref_id, direct_url, path, vul, logger): + url_map = { + "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", + "direct_url": direct_url, + } + + for key, url in url_map.items(): + if url: + try: + ref, created = VulnerabilityReference.objects.update_or_create( + url=url, + defaults={ + "reference_id": ref_id, + "reference_type": VulnerabilityReference.EXPLOIT, + }, + ) + + if created: + VulnerabilityRelatedReference.objects.get_or_create( + vulnerability=vul, + reference=ref, + ) + + except DataError as e: + logger( + f"Failed to Create the Vulnerability Reference For Exploit-DB with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + + +def parse_date(date_string): + if date_string: + try: + date_obj = dateparser.parse(date_string).date() + return date_obj.strftime("%Y-%m-%d") + except (ValueError, TypeError, Exception) as e: + logging.error( + f"Error while parsing ExploitDB date '{date_string}' with error {e!r}:\n{traceback_format_exc()}" + ) + return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py new file mode 100644 index 000000000..e20f61653 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -0,0 +1,96 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from traceback import format_exc as traceback_format_exc + +import requests +from aboutcode.pipeline import LoopProgress + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class VulnerabilityKevPipeline(VulnerableCodePipeline): + """ + Known Exploited Vulnerabilities Pipeline: Retrieve KEV data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + pipeline_id = "enhance_with_kev" + license_expression = None + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_exploits, + ) + + def fetch_exploits(self): + kev_url = "https://raw.githubusercontent.com/aboutcode-org/aboutcode-mirror-kev/refs/heads/main/known_exploited_vulnerabilities.json" + self.log(f"Fetching {kev_url}") + + try: + response = requests.get(kev_url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the KEV Exploits: {kev_url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + self.kev_data = response.json() + + def add_exploits(self): + fetched_exploit_count = self.kev_data.get("count") + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + + for record in progress.iter(self.kev_data.get("vulnerabilities", [])): + vulnerability_exploit_count += add_vulnerability_exploit( + kev_vul=record, + logger=self.log, + ) + + self.log(f"Successfully added {vulnerability_exploit_count:,d} kev exploit") + + +def add_vulnerability_exploit(kev_vul, logger): + cve_id = kev_vul.get("cveID") + + if not cve_id: + return 0 + + vulnerability = None + try: + if alias := Alias.objects.get(alias=cve_id): + vulnerability = alias.vulnerability + except Alias.DoesNotExist: + logger(f"No vulnerability found for aliases {cve_id}") + return 0 + + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="KEV", + defaults={ + "description": kev_vul["shortDescription"], + "date_added": kev_vul["dateAdded"], + "required_action": kev_vul["requiredAction"], + "due_date": kev_vul["dueDate"], + "notes": kev_vul["notes"], + "known_ransomware_campaign_use": True + if kev_vul["knownRansomwareCampaignUse"] == "Known" + else False, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py new file mode 100644 index 000000000..d6989ab47 --- /dev/null +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -0,0 +1,120 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from traceback import format_exc as traceback_format_exc + +import requests +import saneyaml +from aboutcode.pipeline import LoopProgress +from dateutil import parser as dateparser + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class MetasploitImproverPipeline(VulnerableCodePipeline): + """ + Metasploit Exploits Pipeline: Retrieve Metasploit data, iterate through it to identify vulnerabilities + by their associated aliases, and create or update the corresponding Exploit instances. + """ + + pipeline_id = "enhance_with_metasploit" + spdx_license_expression = "BSD-3-clause" + + @classmethod + def steps(cls): + return ( + cls.fetch_exploits, + cls.add_vulnerability_exploits, + ) + + def fetch_exploits(self): + url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" + self.log(f"Fetching {url}") + try: + response = requests.get(url) + response.raise_for_status() + except requests.exceptions.HTTPError as http_err: + self.log( + f"Failed to fetch the Metasploit Exploits: {url} with error {http_err!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + raise + + self.metasploit_data = response.json() + + def add_vulnerability_exploits(self): + fetched_exploit_count = len(self.metasploit_data) + self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") + + vulnerability_exploit_count = 0 + progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) + for _, record in progress.iter(self.metasploit_data.items()): + vulnerability_exploit_count += add_vulnerability_exploit( + record=record, + logger=self.log, + ) + self.log(f"Successfully added {vulnerability_exploit_count:,d} vulnerability exploit") + + +def add_vulnerability_exploit(record, logger): + vulnerabilities = set() + references = record.get("references", []) + + interesting_references = [ + ref for ref in references if not ref.startswith("OSVDB") and not ref.startswith("URL-") + ] + + if not interesting_references: + return 0 + + for ref in interesting_references: + try: + if alias := Alias.objects.get(alias=ref): + vulnerabilities.add(alias.vulnerability) + except Alias.DoesNotExist: + continue + + if not vulnerabilities: + logger(f"No vulnerability found for aliases {interesting_references}") + return 0 + + description = record.get("description", "") + notes = record.get("notes", {}) + platform = record.get("platform") + + source_url = "" + if path := record.get("path"): + source_url = f"https://github.com/rapid7/metasploit-framework/tree/master{path}" + source_date_published = None + + if disclosure_date := record.get("disclosure_date"): + try: + source_date_published = dateparser.parse(disclosure_date).date() + except ValueError as e: + logger( + f"Error while parsing date {disclosure_date} with error {e!r}:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + + for vulnerability in vulnerabilities: + Exploit.objects.update_or_create( + vulnerability=vulnerability, + data_source="Metasploit", + defaults={ + "description": description, + "notes": saneyaml.dump(notes), + "source_date_published": source_date_published, + "platform": platform, + "source_url": source_url, + }, + ) + return 1 diff --git a/vulnerabilities/pipelines/flag_ghost_packages.py b/vulnerabilities/pipelines/flag_ghost_packages.py new file mode 100644 index 000000000..7daee4115 --- /dev/null +++ b/vulnerabilities/pipelines/flag_ghost_packages.py @@ -0,0 +1,104 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from itertools import groupby +from traceback import format_exc as traceback_format_exc + +from aboutcode.pipeline import LoopProgress +from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS +from fetchcode.package_versions import versions +from packageurl import PackageURL + +from vulnerabilities.models import Package +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class FlagGhostPackagePipeline(VulnerableCodePipeline): + """Detect and flag packages that do not exist upstream.""" + + pipeline_id = "flag_ghost_packages" + + @classmethod + def steps(cls): + return (cls.flag_ghost_packages,) + + def flag_ghost_packages(self): + detect_and_flag_ghost_packages(logger=self.log) + + +def detect_and_flag_ghost_packages(logger=None): + """Check if packages are available upstream. If not, mark them as ghost package.""" + interesting_packages_qs = ( + Package.objects.order_by("type", "namespace", "name") + .filter(type__in=FETCHCODE_SUPPORTED_ECOSYSTEMS) + .filter(qualifiers="") + .filter(subpath="") + ) + + distinct_packages_count = ( + interesting_packages_qs.values("type", "namespace", "name") + .distinct("type", "namespace", "name") + .count() + ) + + grouped_packages = groupby( + interesting_packages_qs.paginated(), + key=lambda pkg: (pkg.type, pkg.namespace, pkg.name), + ) + + ghost_package_count = 0 + progress = LoopProgress(total_iterations=distinct_packages_count, logger=logger) + for type_namespace_name, packages in progress.iter(grouped_packages): + ghost_package_count += flag_ghost_packages( + base_purl=PackageURL(*type_namespace_name), + packages=packages, + logger=logger, + ) + + if logger: + logger(f"Successfully flagged {ghost_package_count:,d} ghost Packages") + + +def flag_ghost_packages(base_purl, packages, logger=None): + """ + Check if `packages` are available upstream. + If not, update `is_ghost` to `True`. + Return the number of packages flagged as ghost. + """ + known_versions = get_versions(purl=base_purl, logger=logger) + # Skip if encounter error while fetching known versions + if known_versions is None: + return 0 + + ghost_packages = 0 + for pkg in packages: + pkg.is_ghost = False + if pkg.version.lstrip("vV") not in known_versions: + pkg.is_ghost = True + ghost_packages += 1 + + if logger: + logger(f"Flagging ghost package {pkg.purl!s}", level=logging.DEBUG) + pkg.save() + + return ghost_packages + + +def get_versions(purl, logger=None): + """Return set of known versions for the given purl.""" + try: + return {v.value.lstrip("vV") for v in versions(str(purl))} + except Exception as e: + if logger: + logger( + f"Error while fetching known versions for {purl!s}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return diff --git a/vulnerabilities/importers/github.py b/vulnerabilities/pipelines/github_importer.py similarity index 59% rename from vulnerabilities/importers/github.py rename to vulnerabilities/pipelines/github_importer.py index c12c43044..66c457824 100644 --- a/vulnerabilities/importers/github.py +++ b/vulnerabilities/pipelines/github_importer.py @@ -3,12 +3,15 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging +from traceback import format_exc as traceback_format_exc +from typing import Callable from typing import Iterable +from typing import List from typing import Optional from cwe2.database import Database @@ -21,85 +24,105 @@ from vulnerabilities import utils from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import dedupe from vulnerabilities.utils import get_cwe_id from vulnerabilities.utils import get_item -logger = logging.getLogger(__name__) - -PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM = { - "MAVEN": "maven", - "NUGET": "nuget", - "COMPOSER": "composer", - "PIP": "pypi", - "RUBYGEMS": "gem", - "NPM": "npm", - "RUST": "cargo", - # "GO": "golang", -} - -GITHUB_ECOSYSTEM_BY_PACKAGE_TYPE = { - value: key for (key, value) in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items() -} - -# TODO: We will try to gather more info from GH API -# Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885 -# Check https://github.com/nexB/vulnerablecode/issues/645 -# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'} -# second '%s' is interesting, it will have the value '' for the first request, -GRAPHQL_QUERY_TEMPLATE = """ -query{ - securityVulnerabilities(first: 100, ecosystem: %s, %s) { - edges { - node { - advisory { - identifiers { - type - value - } - summary - references { - url - } - severity - cwes(first: 10){ - nodes { - cweId + +class GitHubAPIImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect GitHub advisories.""" + + pipeline_id = "github_importer" + + spdx_license_expression = "CC-BY-4.0" + license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" + importer_name = "GHSA Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + package_type_by_github_ecosystem = { + "MAVEN": "maven", + "NUGET": "nuget", + "COMPOSER": "composer", + "PIP": "pypi", + "RUBYGEMS": "gem", + "NPM": "npm", + "RUST": "cargo", + # "GO": "golang", + } + + def advisories_count(self): + advisory_query = """ + query{ + securityVulnerabilities(first: 0, ecosystem: %s) { + totalCount + } + } + """ + advisory_counts = 0 + for ecosystem in self.package_type_by_github_ecosystem.keys(): + graphql_query = {"query": advisory_query % (ecosystem)} + response = utils.fetch_github_graphql_query(graphql_query) + advisory_counts += get_item(response, "data", "securityVulnerabilities", "totalCount") + return advisory_counts + + def collect_advisories(self) -> Iterable[AdvisoryData]: + + # TODO: We will try to gather more info from GH API + # Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885 + # Check https://github.com/nexB/vulnerablecode/issues/645 + # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'} + # second '%s' is interesting, it will have the value '' for the first request, + advisory_query = """ + query{ + securityVulnerabilities(first: 100, ecosystem: %s, %s) { + edges { + node { + advisory { + identifiers { + type + value + } + summary + references { + url + } + severity + cwes(first: 10){ + nodes { + cweId + } + } + publishedAt + } + firstPatchedVersion{ + identifier } + package { + name + } + vulnerableVersionRange } - publishedAt - } - firstPatchedVersion{ - identifier } - package { - name + pageInfo { + hasNextPage + endCursor } - vulnerableVersionRange } } - pageInfo { - hasNextPage - endCursor - } - } -} -""" - - -class GitHubAPIImporter(Importer): - spdx_license_expression = "CC-BY-4.0" - importer_name = "GHSA Importer" - license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" - - def advisory_data(self) -> Iterable[AdvisoryData]: - for ecosystem, package_type in PACKAGE_TYPE_BY_GITHUB_ECOSYSTEM.items(): + """ + for ecosystem, package_type in self.package_type_by_github_ecosystem.items(): end_cursor_exp = "" while True: - graphql_query = {"query": GRAPHQL_QUERY_TEMPLATE % (ecosystem, end_cursor_exp)} + graphql_query = {"query": advisory_query % (ecosystem, end_cursor_exp)} response = utils.fetch_github_graphql_query(graphql_query) page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") @@ -114,7 +137,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]: break -def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: +def get_purl(pkg_type: str, github_name: str, logger: Callable = None) -> Optional[PackageURL]: """ Return a PackageURL by splitting the `github_name` using the `pkg_type` convention. Return None and log an error if we can not split or it is an @@ -129,7 +152,8 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: """ if pkg_type == "maven": if ":" not in github_name: - logger.error(f"get_purl: Invalid maven package name {github_name}") + if logger: + logger(f"get_purl: Invalid maven package name {github_name}", level=logging.ERROR) return ns, _, name = github_name.partition(":") return PackageURL(type=pkg_type, namespace=ns, name=name) @@ -143,18 +167,23 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]: if pkg_type in ("nuget", "pypi", "gem", "golang", "npm", "cargo"): return PackageURL(type=pkg_type, name=github_name) - logger.error(f"get_purl: Unknown package type {pkg_type}") + if logger: + logger(f"get_purl: Unknown package type {pkg_type}", level=logging.ERROR) -def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: +def process_response( + resp: dict, package_type: str, logger: Callable = None +) -> Iterable[AdvisoryData]: """ Yield `AdvisoryData` by taking `resp` and `ecosystem` as input """ vulnerabilities = get_item(resp, "data", "securityVulnerabilities", "edges") or [] if not vulnerabilities: - logger.error( - f"No vulnerabilities found for package_type: {package_type!r} in response: {resp!r}" - ) + if logger: + logger( + f"No vulnerabilities found for package_type: {package_type!r} in response: {resp!r}", + level=logging.ERROR, + ) return for vulnerability in vulnerabilities: @@ -162,12 +191,14 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: affected_packages = [] github_advisory = get_item(vulnerability, "node") if not github_advisory: - logger.error(f"No node found in {vulnerability!r}") + if logger: + logger(f"No node found in {vulnerability!r}", level=logging.ERROR) continue advisory = get_item(github_advisory, "advisory") if not advisory: - logger.error(f"No advisory found in {github_advisory!r}") + if logger: + logger(f"No advisory found in {github_advisory!r}", level=logging.ERROR) continue summary = get_item(advisory, "summary") or "" @@ -183,7 +214,7 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: name = get_item(github_advisory, "package", "name") if name: - purl = get_purl(pkg_type=package_type, github_name=name) + purl = get_purl(pkg_type=package_type, github_name=name, logger=logger) if purl: affected_range = get_item(github_advisory, "vulnerableVersionRange") fixed_version = get_item(github_advisory, "firstPatchedVersion", "identifier") @@ -193,7 +224,11 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: package_type, affected_range ) except Exception as e: - logger.error(f"Could not parse affected range {affected_range!r} {e!r}") + if logger: + logger( + f"Could not parse affected range {affected_range!r} {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) affected_range = None if fixed_version: try: @@ -201,7 +236,11 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: fixed_version ) except Exception as e: - logger.error(f"Invalid fixed version {fixed_version!r} {e!r}") + if logger: + logger( + f"Invalid fixed version {fixed_version!r} {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) fixed_version = None if affected_range or fixed_version: affected_packages.append( @@ -236,9 +275,13 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: elif identifier_type == "CVE": pass else: - logger.error(f"Unknown identifier type {identifier_type!r} and value {value!r}") + if logger: + logger( + f"Unknown identifier type {identifier_type!r} and value {value!r}", + level=logging.ERROR, + ) - weaknesses = get_cwes_from_github_advisory(advisory) + weaknesses = get_cwes_from_github_advisory(advisory, logger) yield AdvisoryData( aliases=sorted(dedupe(aliases)), @@ -251,7 +294,7 @@ def process_response(resp: dict, package_type: str) -> Iterable[AdvisoryData]: ) -def get_cwes_from_github_advisory(advisory) -> [int]: +def get_cwes_from_github_advisory(advisory, logger=None) -> List[int]: """ Return the cwe-id list from advisory ex: [ 522 ] by extracting the cwe_list from advisory ex: [{'cweId': 'CWE-522'}] @@ -267,6 +310,7 @@ def get_cwes_from_github_advisory(advisory) -> [int]: try: db.get(cwe_id) weaknesses.append(cwe_id) - except Exception: - logger.error("Invalid CWE id") + except Exception as e: + if logger: + logger(f"Invalid CWE id {e!r} \n {traceback_format_exc()}", level=logging.ERROR) return weaknesses diff --git a/vulnerabilities/importers/gitlab.py b/vulnerabilities/pipelines/gitlab_importer.py similarity index 66% rename from vulnerabilities/importers/gitlab.py rename to vulnerabilities/pipelines/gitlab_importer.py index cd42b24ed..4f25c4d94 100644 --- a/vulnerabilities/importers/gitlab.py +++ b/vulnerabilities/pipelines/gitlab_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -12,11 +12,12 @@ from pathlib import Path from typing import Iterable from typing import List -from typing import Optional +from typing import Tuple import pytz import saneyaml from dateutil import parser as dateparser +from fetchcode.vcs import fetch_via_vcs from packageurl import PackageURL from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange @@ -25,58 +26,91 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import build_description from vulnerabilities.utils import get_advisory_url from vulnerabilities.utils import get_cwe_id -logger = logging.getLogger(__name__) -PURL_TYPE_BY_GITLAB_SCHEME = { - "conan": "conan", - "gem": "gem", - # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 - # "go": "golang", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "packagist": "composer", - "pypi": "pypi", -} +class GitLabImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisory from GitLab Advisory Database (Open Source Edition).""" -GITLAB_SCHEME_BY_PURL_TYPE = {v: k for k, v in PURL_TYPE_BY_GITLAB_SCHEME.items()} + pipeline_id = "gitlab_importer" - -class GitLabAPIImporter(Importer): spdx_license_expression = "MIT" license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" importer_name = "GitLab Importer" repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/" - def advisory_data(self, _keep_clone=False) -> Iterable[AdvisoryData]: - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.import_new_advisories, + cls.clean_downloads, + ) - for file_path in base_path.glob("**/*.yml"): - gitlab_type, package_slug, vuln_id = parse_advisory_path( - base_path=base_path, - file_path=file_path, - ) + purl_type_by_gitlab_scheme = { + "conan": "conan", + "gem": "gem", + # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 + # "go": "golang", + "maven": "maven", + "npm": "npm", + "nuget": "nuget", + "packagist": "composer", + "pypi": "pypi", + } + + gitlab_scheme_by_purl_type = {v: k for k, v in purl_type_by_gitlab_scheme.items()} + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + root = Path(self.vcs_response.dest_dir) + return sum(1 for _ in root.rglob("*.yml")) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + base_path = Path(self.vcs_response.dest_dir) + + for file_path in base_path.rglob("*.yml"): + if file_path.parent == base_path: + continue + + gitlab_type, _, _ = parse_advisory_path( + base_path=base_path, + file_path=file_path, + ) - if gitlab_type in PURL_TYPE_BY_GITLAB_SCHEME: - yield parse_gitlab_advisory(file=file_path, base_path=base_path) + if gitlab_type not in self.purl_type_by_gitlab_scheme: + # self.log( + # f"Unknown package type {gitlab_type!r} in {file_path!r}", + # level=logging.ERROR, + # ) + continue + + yield parse_gitlab_advisory( + file=file_path, + base_path=base_path, + gitlab_scheme_by_purl_type=self.gitlab_scheme_by_purl_type, + purl_type_by_gitlab_scheme=self.purl_type_by_gitlab_scheme, + logger=self.log, + ) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() - else: - logger.error(f"Unknow package type {gitlab_type!r} in {file_path!r}") - continue - finally: - if self.vcs_response and not _keep_clone: - self.vcs_response.delete() + def on_failure(self): + self.clean_downloads() -def parse_advisory_path(base_path: Path, file_path: Path) -> Optional[AdvisoryData]: +def parse_advisory_path(base_path: Path, file_path: Path) -> Tuple[str, str, str]: """ Parse a gitlab advisory file and return a 3-tuple of: (gitlab_type, package_slug, vulnerability_id) @@ -96,21 +130,21 @@ def parse_advisory_path(base_path: Path, file_path: Path) -> Optional[AdvisoryDa >>> parse_advisory_path(base_path=base_path, file_path=file_path) ('npm', '@express/beego/beego/v2', 'CVE-2021-43831') """ - relative_path_segments = str(file_path.relative_to(base_path)).strip("/").split("/") + relative_path_segments = file_path.relative_to(base_path).parts gitlab_type = relative_path_segments[0] - vuln_id = relative_path_segments[-1].replace(".yml", "") + vuln_id = file_path.stem package_slug = "/".join(relative_path_segments[1:-1]) return gitlab_type, package_slug, vuln_id -def get_purl(package_slug): +def get_purl(package_slug, purl_type_by_gitlab_scheme, logger): """ Return a PackageURL object from a package slug """ parts = [p for p in package_slug.strip("/").split("/") if p] gitlab_scheme = parts[0] - purl_type = PURL_TYPE_BY_GITLAB_SCHEME[gitlab_scheme] + purl_type = purl_type_by_gitlab_scheme[gitlab_scheme] if gitlab_scheme == "go": name = "/".join(parts[1:]) return PackageURL(type=purl_type, namespace=None, name=name) @@ -125,7 +159,7 @@ def get_purl(package_slug): name = parts[-1] namespace = "/".join(parts[1:-1]) return PackageURL(type=purl_type, namespace=namespace, name=name) - logger.error(f"get_purl: package_slug can not be parsed: {package_slug!r}") + logger(f"get_purl: package_slug can not be parsed: {package_slug!r}", level=logging.ERROR) return @@ -140,7 +174,7 @@ def extract_affected_packages( In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range. Since we can not determine which package fixes which range. We store the all the fixed_versions with the same affected_version_range in the advisory. - Later the advisory data is used to be infered in the GitLabBasicImprover. + Later the advisory data is used to be inferred in the GitLabBasicImprover. """ for fixed_version in fixed_versions: yield AffectedPackage( @@ -150,7 +184,9 @@ def extract_affected_packages( ) -def parse_gitlab_advisory(file, base_path): +def parse_gitlab_advisory( + file, base_path, gitlab_scheme_by_purl_type, purl_type_by_gitlab_scheme, logger +): """ Parse a Gitlab advisory file and return an AdvisoryData or None. These files are YAML. There is a JSON schema documented at @@ -177,8 +213,9 @@ def parse_gitlab_advisory(file, base_path): with open(file) as f: gitlab_advisory = saneyaml.load(f) if not isinstance(gitlab_advisory, dict): - logger.error( - f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}" + logger( + f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}", + level=logging.ERROR, ) return @@ -199,9 +236,15 @@ def parse_gitlab_advisory(file, base_path): base_path=base_path, url="https://gitlab.com/gitlab-org/advisories-community/-/blob/main/", ) - purl: PackageURL = get_purl(package_slug=package_slug) + purl: PackageURL = get_purl( + package_slug=package_slug, + purl_type_by_gitlab_scheme=purl_type_by_gitlab_scheme, + logger=logger, + ) if not purl: - logger.error(f"parse_yaml_file: purl is not valid: {file!r} {package_slug!r}") + logger( + f"parse_yaml_file: purl is not valid: {file!r} {package_slug!r}", level=logging.ERROR + ) return AdvisoryData( aliases=aliases, summary=summary, @@ -214,7 +257,7 @@ def parse_gitlab_advisory(file, base_path): affected_range = gitlab_advisory.get("affected_range") gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"]) vrc: VersionRange = RANGE_CLASS_BY_SCHEMES[purl.type] - gitlab_scheme = GITLAB_SCHEME_BY_PURL_TYPE[purl.type] + gitlab_scheme = gitlab_scheme_by_purl_type[purl.type] try: if affected_range: if gitlab_scheme in gitlab_native_schemes: @@ -224,8 +267,9 @@ def parse_gitlab_advisory(file, base_path): else: affected_version_range = vrc.from_native(affected_range) except Exception as e: - logger.error( - f"parse_yaml_file: affected_range is not parsable: {affected_range!r} type:{purl.type!r} error: {e!r}\n {traceback.format_exc()}" + logger( + f"parse_yaml_file: affected_range is not parsable: {affected_range!r} for: {purl!s} error: {e!r}\n {traceback.format_exc()}", + level=logging.ERROR, ) parsed_fixed_versions = [] @@ -234,8 +278,9 @@ def parse_gitlab_advisory(file, base_path): fixed_version = vrc.version_class(fixed_version) parsed_fixed_versions.append(fixed_version) except Exception as e: - logger.error( - f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}" + logger( + f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}", + level=logging.ERROR, ) if parsed_fixed_versions: diff --git a/vulnerabilities/importers/nginx.py b/vulnerabilities/pipelines/nginx_importer.py similarity index 77% rename from vulnerabilities/importers/nginx.py rename to vulnerabilities/pipelines/nginx_importer.py index 4fe0ca6ae..c5e017033 100644 --- a/vulnerabilities/importers/nginx.py +++ b/vulnerabilities/pipelines/nginx_importer.py @@ -3,58 +3,62 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import logging from typing import Iterable -from typing import List from typing import NamedTuple import requests from bs4 import BeautifulSoup -from django.db.models.query import QuerySet from packageurl import PackageURL from univers.version_range import NginxVersionRange from univers.versions import NginxVersion from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.severity_systems import GENERIC -logger = logging.getLogger(__name__) +class NginxImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect Nginx security advisories.""" -class NginxImporter(Importer): - - url = "https://nginx.org/en/security_advisories.html" + pipeline_id = "nginx_importer" spdx_license_expression = "BSD-2-Clause" license_url = "https://nginx.org/LICENSE" + url = "https://nginx.org/en/security_advisories.html" importer_name = "Nginx Importer" - def advisory_data(self) -> Iterable[AdvisoryData]: - text = self.fetch() - yield from advisory_data_from_text(text) + @classmethod + def steps(cls): + return ( + cls.fetch, + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) def fetch(self): - return requests.get(self.url).content + self.log(f"Fetch `{self.url}`") + self.advisory_data = requests.get(self.url).text + def advisories_count(self): + return self.advisory_data.count("

  • ") -def advisory_data_from_text(text): - """ - Yield AdvisoryData from the ``text`` of the nginx security advisories HTML - web page. - """ - soup = BeautifulSoup(text, features="lxml") - vuln_list = soup.select("li p") - for vuln_info in vuln_list: - ngnix_adv = parse_advisory_data_from_paragraph(vuln_info) - yield to_advisory_data(ngnix_adv) + def collect_advisories(self) -> Iterable[AdvisoryData]: + """ + Yield AdvisoryData from nginx security advisories HTML + web page. + """ + soup = BeautifulSoup(self.advisory_data, features="lxml") + vulnerability_list = soup.select("li p") + for vulnerability_info in vulnerability_list: + ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info) + yield to_advisory_data(ngnix_advisory) class NginxAdvisory(NamedTuple): @@ -69,7 +73,7 @@ def to_dict(self): return self._asdict() -def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: +def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryData: """ Return AdvisoryData from an NginxAdvisory tuple. """ @@ -77,7 +81,7 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: package_type = "nginx" qualifiers = {} - _, _, affected_version_range = ngnx_adv.vulnerable.partition(":") + _, _, affected_version_range = nginx_adv.vulnerable.partition(":") if "nginx/Windows" in affected_version_range: qualifiers["os"] = "windows" affected_version_range = affected_version_range.replace("nginx/Windows", "") @@ -87,7 +91,7 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: affected_version_range = NginxVersionRange.from_native(affected_version_range) affected_packages = [] - _, _, fixed_versions = ngnx_adv.not_vulnerable.partition(":") + _, _, fixed_versions = nginx_adv.not_vulnerable.partition(":") for fixed_version in fixed_versions.split(","): fixed_version = fixed_version.rstrip("+") @@ -112,17 +116,17 @@ def to_advisory_data(ngnx_adv: NginxAdvisory) -> AdvisoryData: ) return AdvisoryData( - aliases=ngnx_adv.aliases, - summary=ngnx_adv.summary, + aliases=nginx_adv.aliases, + summary=nginx_adv.summary, affected_packages=affected_packages, - references=ngnx_adv.references, + references=nginx_adv.references, url="https://nginx.org/en/security_advisories.html", ) -def parse_advisory_data_from_paragraph(vuln_info): +def parse_advisory_data_from_paragraph(vulnerability_info): """ - Return an NginxAdvisory from a ``vuln_info`` bs4 paragraph. + Return an NginxAdvisory from a ``vulnerability_info`` bs4 paragraph. An advisory paragraph, without html markup, looks like this: @@ -145,7 +149,7 @@ def parse_advisory_data_from_paragraph(vuln_info): # we iterate on the children to accumulate values in variables # FIXME: using an explicit xpath-like query could be simpler - for child in vuln_info.children: + for child in vulnerability_info.children: if is_first: summary = child is_first = False diff --git a/vulnerabilities/importers/npm.py b/vulnerabilities/pipelines/npm_importer.py similarity index 78% rename from vulnerabilities/importers/npm.py rename to vulnerabilities/pipelines/npm_importer.py index 4dcc30705..7b6d3aba2 100644 --- a/vulnerabilities/importers/npm.py +++ b/vulnerabilities/pipelines/npm_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -11,44 +11,58 @@ from pathlib import Path from typing import Iterable -from typing import List import pytz from dateutil.parser import parse +from fetchcode.vcs import fetch_via_vcs from packageurl import PackageURL from univers.version_range import NpmVersionRange from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.severity_systems import CVSSV2 from vulnerabilities.severity_systems import CVSSV3 from vulnerabilities.utils import build_description from vulnerabilities.utils import load_json -class NpmImporter(Importer): +class NpmImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from nodejs GitHub repository.""" + + pipeline_id = "npm_importer" + spdx_license_expression = "MIT" license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" repo_url = "git+https://github.com/nodejs/security-wg" importer_name = "Npm Importer" - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(self.repo_url) - path = Path(self.vcs_response.dest_dir) + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.import_new_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) - vuln = path / "vuln" - npm_vulns = vuln / "npm" - for file in npm_vulns.glob("*.json"): - yield from self.to_advisory_data(file) - finally: - if self.vcs_response: - self.vcs_response.delete() + def advisories_count(self): + vuln_directory = Path(self.vcs_response.dest_dir) / "vuln" / "npm" + return sum(1 for _ in vuln_directory.glob("*.json")) - def to_advisory_data(self, file: Path) -> List[AdvisoryData]: + def collect_advisories(self) -> Iterable[AdvisoryData]: + vuln_directory = Path(self.vcs_response.dest_dir) / "vuln" / "npm" + + for advisory in vuln_directory.glob("*.json"): + yield from self.to_advisory_data(advisory) + + def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: data = load_json(file) id = data.get("id") description = data.get("overview") or "" @@ -144,3 +158,11 @@ def get_affected_package(self, data, package_name): affected_version_range=affected_version_range, fixed_version=fixed_version, ) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() diff --git a/vulnerabilities/importers/nvd.py b/vulnerabilities/pipelines/nvd_importer.py similarity index 88% rename from vulnerabilities/importers/nvd.py rename to vulnerabilities/pipelines/nvd_importer.py index 1a6048dfd..bd6f33cf9 100644 --- a/vulnerabilities/importers/nvd.py +++ b/vulnerabilities/pipelines/nvd_importer.py @@ -3,13 +3,16 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import gzip import json +import logging from datetime import date +from traceback import format_exc as traceback_format_exc +from typing import Iterable import attr import requests @@ -17,14 +20,18 @@ from vulnerabilities import severity_systems from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.utils import get_cwe_id from vulnerabilities.utils import get_item -class NVDImporter(Importer): +class NVDImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from NVD.""" + + pipeline_id = "nvd_importer" + # See https://github.com/nexB/vulnerablecode/issues/665 for follow up spdx_license_expression = ( "LicenseRef-scancode-us-govt-public-domain AND LicenseRef-scancode-cve-tou" @@ -61,19 +68,46 @@ class NVDImporter(Importer): """ importer_name = "NVD Importer" - def advisory_data(self): - for _year, cve_data in fetch_cve_data_1_1(): + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self): + url = "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=1" + + advisory_count = 0 + try: + response = requests.get(url) + response.raise_for_status() + data = response.json() + except requests.HTTPError as http_err: + self.log( + f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + return advisory_count + + advisory_count = data.get("totalResults", 0) + return advisory_count + + def collect_advisories(self) -> Iterable[AdvisoryData]: + for _year, cve_data in fetch_cve_data_1_1(logger=self.log): yield from to_advisories(cve_data=cve_data) # Isolating network calls for simplicity of testing -def fetch(url): +def fetch(url, logger=None): + if logger: + logger(f"Fetching `{url}`") gz_file = requests.get(url) data = gzip.decompress(gz_file.content) return json.loads(data) -def fetch_cve_data_1_1(starting_year=2002): +def fetch_cve_data_1_1(starting_year=2002, logger=None): """ Yield tuples of (year, lists of CVE mappings) from the NVD, one for each year since ``starting_year`` defaulting to 2002. @@ -82,7 +116,7 @@ def fetch_cve_data_1_1(starting_year=2002): # NVD json feeds start from 2002. for year in range(starting_year, current_year + 1): download_url = f"https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-{year}.json.gz" - yield year, fetch(url=download_url) + yield year, fetch(url=download_url, logger=logger) def to_advisories(cve_data): diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py new file mode 100644 index 000000000..aebafacf4 --- /dev/null +++ b/vulnerabilities/pipelines/pypa_importer.py @@ -0,0 +1,73 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +from typing import Iterable + +import saneyaml +from fetchcode.vcs import fetch_via_vcs + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.utils import get_advisory_url + + +class PyPaImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from PyPA GitHub repository.""" + + pipeline_id = "pypa_importer" + + spdx_license_expression = "CC-BY-4.0" + license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" + repo_url = "git+https://github.com/pypa/advisory-database" + importer_name = "Pypa Importer" + + @classmethod + def steps(cls): + return ( + cls.clone, + cls.collect_and_store_advisories, + cls.import_new_advisories, + cls.clean_downloads, + ) + + def clone(self): + self.log(f"Cloning `{self.repo_url}`") + self.vcs_response = fetch_via_vcs(self.repo_url) + + def advisories_count(self): + vulns_directory = Path(self.vcs_response.dest_dir) / "vulns" + return sum(1 for _ in vulns_directory.rglob("*.yaml")) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + from vulnerabilities.importers.osv import parse_advisory_data + + base_directory = Path(self.vcs_response.dest_dir) + vulns_directory = base_directory / "vulns" + + for advisory in vulns_directory.rglob("*.yaml"): + advisory_url = get_advisory_url( + file=advisory, + base_path=base_directory, + url="https://github.com/pypa/advisory-database/blob/main/", + ) + advisory_dict = saneyaml.load(advisory.read_text()) + yield parse_advisory_data( + raw_data=advisory_dict, + supported_ecosystems=["pypi"], + advisory_url=advisory_url, + ) + + def clean_downloads(self): + if self.vcs_response: + self.log(f"Removing cloned repository") + self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py new file mode 100644 index 000000000..32a9fd896 --- /dev/null +++ b/vulnerabilities/pipelines/pysec_importer.py @@ -0,0 +1,66 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +import logging +from io import BytesIO +from typing import Iterable +from zipfile import ZipFile + +import requests + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + + +class PyPIImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from PyPI.""" + + pipeline_id = "pysec_importer" + + license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" + url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" + spdx_license_expression = "CC-BY-4.0" + importer_name = "PyPI Importer" + + @classmethod + def steps(cls): + return ( + cls.fetch_zip, + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def fetch_zip(self): + self.log(f"Fetching `{self.url}`") + self.advisory_zip = requests.get(self.url).content + + def advisories_count(self) -> int: + with ZipFile(BytesIO(self.advisory_zip)) as zip: + advisory_count = sum(1 for file in zip.namelist() if file.startswith("PYSEC-")) + return advisory_count + + def collect_advisories(self) -> Iterable[AdvisoryData]: + """Yield AdvisoryData using a zipped data dump of OSV data""" + from vulnerabilities.importers.osv import parse_advisory_data + + with ZipFile(BytesIO(self.advisory_zip)) as zip_file: + for file_name in zip_file.namelist(): + if not file_name.startswith("PYSEC-"): + self.log( + f"Unsupported PyPI advisory data file: {file_name}", + level=logging.ERROR, + ) + continue + with zip_file.open(file_name) as f: + vul_info = json.load(f) + yield parse_advisory_data( + raw_data=vul_info, + supported_ecosystems=["pypi"], + advisory_url=self.url, + ) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py new file mode 100644 index 000000000..6637122a3 --- /dev/null +++ b/vulnerabilities/pipes/advisory.py @@ -0,0 +1,164 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging +from datetime import datetime +from datetime import timezone +from traceback import format_exc as traceback_format_exc +from typing import Callable + +from django.db import transaction + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.improver import MAX_CONFIDENCE +from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import FixingPackageRelatedVulnerability +from vulnerabilities.models import Package +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness + + +def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = None): + obj = None + try: + obj, _ = Advisory.objects.get_or_create( + aliases=advisory.aliases, + summary=advisory.summary, + affected_packages=[pkg.to_dict() for pkg in advisory.affected_packages], + references=[ref.to_dict() for ref in advisory.references], + date_published=advisory.date_published, + weaknesses=advisory.weaknesses, + url=advisory.url, + defaults={ + "created_by": pipeline_id, + "date_collected": datetime.now(timezone.utc), + }, + ) + except Exception as e: + if logger: + logger( + f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}", + level=logging.ERROR, + ) + + return obj + + +@transaction.atomic +def import_advisory( + advisory: Advisory, + pipeline_id: str, + confidence: int = MAX_CONFIDENCE, + logger: Callable = None, +): + """ + Create initial Vulnerability Package relationships for the advisory, + including references and severity scores. + + Package relationships are established only for resolved (concrete) versions. + """ + from vulnerabilities import import_runner + from vulnerabilities.improvers import default + + advisory_data: AdvisoryData = advisory.to_advisory_data() + if logger: + logger(f"Importing advisory id: {advisory.id}", level=logging.DEBUG) + + affected_purls = [] + fixed_purls = [] + for affected_package in advisory_data.affected_packages: + package_affected_purls, package_fixed_purls = default.get_exact_purls( + affected_package=affected_package + ) + affected_purls.extend(package_affected_purls) + fixed_purls.extend(package_fixed_purls) + + vulnerability = import_runner.get_or_create_vulnerability_and_aliases( + vulnerability_id=None, + aliases=advisory_data.aliases, + summary=advisory_data.summary, + advisory=advisory, + ) + + if not vulnerability: + if logger: + logger(f"Unable to get vulnerability for advisory: {advisory!r}", level=logging.ERROR) + return + + for ref in advisory_data.references: + reference = VulnerabilityReference.objects.get_or_none( + reference_id=ref.reference_id, + url=ref.url, + ) + if not reference: + reference = import_runner.create_valid_vulnerability_reference( + reference_id=ref.reference_id, + url=ref.url, + ) + if reference: + VulnerabilityRelatedReference.objects.update_or_create( + reference=reference, + vulnerability=vulnerability, + ) + for severity in ref.severities: + try: + published_at = str(severity.published_at) if severity.published_at else None + vulnerability_severity, created = VulnerabilitySeverity.objects.update_or_create( + scoring_system=severity.system.identifier, + url=ref.url, + value=severity.value, + scoring_elements=severity.scoring_elements, + defaults={ + "published_at": published_at, + }, + ) + vulnerability.severities.add(vulnerability_severity) + except: + if logger: + logger( + f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}", + level=logging.ERROR, + ) + if not created: + if logger: + logger( + f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " + f"and scoring_elements: {severity.scoring_elements!r}", + level=logging.DEBUG, + ) + + for affected_purl in affected_purls or []: + vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) + AffectedByPackageRelatedVulnerability( + vulnerability=vulnerability, + package=vulnerable_package, + created_by=pipeline_id, + confidence=confidence, + ).update_or_create(advisory=advisory) + + for fixed_purl in fixed_purls: + fixed_package, _ = Package.objects.get_or_create_from_purl(purl=fixed_purl) + FixingPackageRelatedVulnerability( + vulnerability=vulnerability, + package=fixed_package, + created_by=pipeline_id, + confidence=confidence, + ).update_or_create(advisory=advisory) + + if advisory_data.weaknesses and vulnerability: + for cwe_id in advisory_data.weaknesses: + cwe_obj, _ = Weakness.objects.get_or_create(cwe_id=cwe_id) + cwe_obj.vulnerabilities.add(vulnerability) + cwe_obj.save() + + advisory.date_imported = datetime.now(timezone.utc) + advisory.save() diff --git a/vulnerabilities/references.py b/vulnerabilities/references.py index 87b45b9f6..47225f520 100644 --- a/vulnerabilities/references.py +++ b/vulnerabilities/references.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py new file mode 100644 index 000000000..a4508a03f --- /dev/null +++ b/vulnerabilities/risk.py @@ -0,0 +1,114 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from urllib.parse import urlparse + +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.weight_config import WEIGHT_CONFIG + +DEFAULT_WEIGHT = 5 + + +def get_weighted_severity(severities): + """ + Weighted Severity is the maximum value obtained when each Severity is multiplied + by its associated Weight/10. + Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 + """ + if not severities: + return 0 + + score_map = { + "low": 3, + "moderate": 6.9, + "medium": 6.9, + "high": 8.9, + "important": 8.9, + "critical": 10.0, + "urgent": 10.0, + } + + score_list = [] + for severity in severities: + parsed_url = urlparse(severity.url) + severity_source = parsed_url.netloc.replace("www.", "", 1) + weight = WEIGHT_CONFIG.get(severity_source, DEFAULT_WEIGHT) + max_weight = float(weight) / 10 + vul_score = severity.value + try: + vul_score = float(vul_score) + vul_score_value = vul_score * max_weight + except ValueError: + vul_score = vul_score.lower() + vul_score_value = score_map.get(vul_score, 0) * max_weight + + score_list.append(vul_score_value) + + max_score = max(score_list) if score_list else 0 + return round(max_score, 1) + + +def get_exploitability_level(exploits, references, severities): + """ + Exploitability refers to the potential or + probability of a software package vulnerability being exploited by + malicious actors to compromise systems, applications, or networks. + It is determined automatically by discovery of exploits. + """ + # no exploit known ( default .5) + exploit_level = 0.5 + + if exploits: + # Automatable Exploit with PoC script published OR known exploits (KEV) in the wild OR known ransomware + exploit_level = 2 + + elif severities: + # high EPSS. + for severity in severities: + if severity.scoring_system == EPSS.identifier and float(severity.value) > 0.8: + exploit_level = 2 + break + + elif references: + # PoC/Exploit script published + for reference in references: + if reference.reference_type == VulnerabilityReference.EXPLOIT: + exploit_level = 1 + break + + return exploit_level + + +def compute_vulnerability_risk_factors(references, severities, exploits): + """ + Risk may be expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + + Risk = min(weighted severity * exploitability, 10) + """ + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return weighted_severity, exploitability + + +def compute_package_risk(package): + """ + Calculate the risk for a package by iterating over all vulnerabilities that affects this package + and determining the associated risk. + """ + result = [] + for relation in package.affectedbypackagerelatedvulnerability_set.all(): + if risk := relation.vulnerability.risk_score: + result.append(float(risk)) + + if not result: + return + + return round(max(result), 1) diff --git a/vulnerabilities/rpm_utils.py b/vulnerabilities/rpm_utils.py index 206f3668f..84d440c9f 100644 --- a/vulnerabilities/rpm_utils.py +++ b/vulnerabilities/rpm_utils.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index f5be70a5b..946cb6479 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 632790304..dd72d0500 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -2,6 +2,7 @@ {% load humanize %} {% load widget_tweaks %} {% load static %} +{% load url_filters %} {% block title %} VulnerableCode Package Details - {{ package.purl }} @@ -62,6 +63,21 @@ {{ fixed_package_details.purl.to_string }} + {% if package.is_ghost %} + + + Tags + + + + Ghost + + + + {% endif %} @@ -76,7 +92,7 @@ {% if fixed_package_details.next_non_vulnerable.version %} - {{ fixed_package_details.next_non_vulnerable.version }} {% else %} None. @@ -89,13 +105,27 @@ {% if fixed_package_details.latest_non_vulnerable.version %} - {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} None. {% endif %} + + + + Risk + + + + {% if package.risk_score %} + {{ package.risk_score }} + {% endif %} + + @@ -104,7 +134,7 @@

    - Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
    @@ -112,7 +142,7 @@ - + @@ -145,32 +175,28 @@ {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} {% if vuln.fixed_by_package_details is None %} - There are no reported fixed by versions. + There are no reported fixed by versions. {% else %} {% for fixed_pkg in vuln.fixed_by_package_details %}
    {% if fixed_pkg.fixed_by_purl_vulnerabilities|length == 0 %} - {{ fixed_pkg.fixed_by_purl.version }}
    - Affected - by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} - {{ fixed_pkg.fixed_by_purl.version }} {% if fixed_pkg.fixed_by_purl_vulnerabilities|length != 1 %}
    - Affected - by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
    - Affected - by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %} - +
    - This version is affected - by these other vulnerabilities: + This version is affected by these other vulnerabilities:
    {% for fixed_by_vuln in fixed_pkg.fixed_by_purl_vulnerabilities %}
    @@ -211,8 +236,7 @@ {% empty %}
    {% endfor %} @@ -222,7 +246,7 @@
    - Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
    Vulnerability SummaryFixed byFixed by
    - This package is not known to be affected by vulnerabilities. + This package is not known to be affected by vulnerabilities.
    @@ -258,8 +282,7 @@ {% empty %} {% endfor %} @@ -325,4 +348,4 @@ {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 2f91a5422..1f7687429 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -41,14 +41,14 @@ - Affected by vulnerabilities + Affected by vulnerabilities diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index bdada6ee1..023d3f97f 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -32,8 +32,8 @@ - - + + diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index c950adad1..e9e58c79e 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -3,6 +3,7 @@ {% load widget_tweaks %} {% load static %} {% load show_cvss %} +{% load url_filters %} {% block title %} VulnerableCode Vulnerability Details - {{ vulnerability.vulnerability_id }} @@ -32,17 +33,10 @@ Essentials -
  • +
  • - Fixed by packages ({{ fixed_by_packages|length }}) - - -
    • -
  • - - - Affected packages ({{ affected_packages|length }}) + Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    • @@ -61,11 +55,11 @@ - {% if vulnerability.kev %} -
  • + {% if vulnerability.exploits %} +
  • - Known Exploited Vulnerabilities + Exploits ({{ vulnerability.exploits.count }})
    • @@ -77,7 +71,7 @@ EPSS - +
  • @@ -127,6 +121,38 @@
    • + + + + + + + + + + + + + + + +
    - This package is not known to fix - vulnerabilities. + This package is not known to fix vulnerabilities.
    - Fixing vulnerabilities + Fixing vulnerabilities
    Vulnerability id AliasesAffected packagesFixed by packagesAffected packagesFixed by packages
    Status {{ status }}
    + Exploitability + {{ vulnerability.exploitability }} +
    Weighted Severity + {{ vulnerability.weighted_severity }} +
    Risk + {{ vulnerability.risk_score }} +
    @@ -145,13 +171,9 @@ {{ severity.scoring_system }} {{ severity.value }} - {% if severity.reference.url %} - - {{ severity.reference.url }} + + {{ severity.url }} - {% else %} - {{ severity.reference.reference_id }} - {% endif %} {% empty %} @@ -164,62 +186,52 @@
    - Fixed by packages ({{ fixed_by_packages|length }}) -
    -
    - - {% for package in fixed_by_packages|slice:":3" %} - - - - {% empty %} - - - - {% endfor %} - {% if fixed_by_packages|length > 3 %} - - - - {% endif %} -
    - {{ package.purl }} -
    -
    - There are no known fixed by packages. -
    - See Fixed - by packages tab for more -
    -
    - -
    - Affected packages ({{ affected_packages|length }}) + Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    - {% for package in affected_packages|slice:":3" %} - - - - {% empty %} - - - - {% endfor %} - {% if affected_packages|length > 3 %} - - - - {% endif %} + + + + + + + + {% for package in affected_packages|slice:":3" %} + + + + + {% empty %} + + + + {% endfor %} + {% if affected_packages|length > 3 %} + + + + {% endif %} +
    - {{ package.purl }} -
    -
    - There are no known affected packages. -
    - See Affected packages tab for more -
    AffectedFixed by
    + {{ package.purl }} + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} +
    + This vulnerability is not known to affect any packages. +
    + See Affected/Fixed by packages tab for more +
    @@ -251,6 +263,48 @@ +
    + + + + + + + + + {% for package in affected_packages %} + + + + + {% empty %} + + + + {% endfor %} + +
    AffectedFixed by
    + {{ package.purl }} + + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} + +
    + This vulnerability is not known to affect any packages. +
    +
    +
    @@ -287,63 +341,6 @@
    -
    - - - - - - - - {% for package in affected_packages %} - - - - {% empty %} - - - - {% endfor %} - -
    - Package URL -
    - {{ package.purl }} -
    - This vulnerability is not known to affect any packages. -
    -
    - -
    - - - - - - - - {% for package in fixed_by_packages %} - - - - {% empty %} - - - - {% endfor %} - -
    - Package URL -
    - {{package.purl }} -
    - This vulnerability is not known to be fixed by any packages. -
    -
    {% for severity_vector in severity_vectors %} {% if severity_vector.version == '2.0' %} @@ -439,87 +436,157 @@ {% endfor %}
    - {% if vulnerability.kev %} -
    -
    - Known Exploited Vulnerabilities -
    - + + +
    + {% for exploit in vulnerability.exploits.all %} +
    + + + + + - - - - - - {% if vulnerability.kev.description %} + {% if exploit.date_added %} - + {% endif %} - {% if vulnerability.kev.required_action %} + {% if exploit.description %} - + {% endif %} - - {% if vulnerability.kev.resources_and_notes %} + {% if exploit.required_action %} - + {% endif %} - - {% if vulnerability.kev.due_date %} + {% if exploit.due_date %} - + {% endif %} - {% if vulnerability.kev.date_added %} + {% if exploit.notes %} + + + + + {% endif %} + {% if exploit.known_ransomware_campaign_use is not None %} + + + + + {% endif %} + {% if exploit.source_date_published %} + + + + + {% endif %} + {% if exploit.exploit_type %} + + + + + {% endif %} + {% if exploit.platform %} + + + + + {% endif %} + {% if exploit.source_date_updated %} + + + + + {% endif %} + + {% if exploit.source_url %} - + {% endif %} - -
    Data source {{ exploit.data_source }}
    - - Known Ransomware Campaign Use: - - {{ vulnerability.kev.get_known_ransomware_campaign_use_type }}
    - Description: + data-tooltip="The date the vulnerability was added to an exploit catalog."> + Date added {{ vulnerability.kev.description }}{{ exploit.date_added }}
    - Required Action: + data-tooltip="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description"> + Description {{ vulnerability.kev.required_action }}{{ exploit.description }}
    - Notes: + data-tooltip="The required action to address the vulnerability, + typically to apply vendor updates or apply vendor mitigations or to discontinue use."> + Required action {{ vulnerability.kev.resources_and_notes }}{{ exploit.required_action }}
    - Due Date: + data-tooltip="The date the required action is due in the format YYYY-MM-DD, + which applies to all USA federal civilian executive branch (FCEB) agencies, + but all organizations are strongly encouraged to execute the required action."> + Due date {{ vulnerability.kev.due_date }}{{ exploit.due_date }}
    + + Note + + 
    {{ exploit.notes }}
    + + Ransomware campaign use + + {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    + + Source publication date + + {{ exploit.source_date_published }}
    + + Exploit type + + {{ exploit.exploit_type }}
    + + Platform + + {{ exploit.platform }}
    + + Source update date + + {{ exploit.source_date_updated }}
    - Date Added: + data-tooltip="The URL to the exploit as provided in the original upstream data source."> + Source URL {{ vulnerability.kev.date_added }}{{ exploit.source_url }}
    -
    - {% endif %} + + {% empty %} + + + No exploits are available. + + + {% endfor %} + + {% for severity in severities %} {% if severity.scoring_system == 'epss' %} @@ -533,7 +600,7 @@ - Percentile: + Percentile {{ severity.scoring_elements }} @@ -543,24 +610,24 @@ - EPSS score: + EPSS score {{ severity.value }} - + {% if severity.published_at %} - Published at: + Published at {{ severity.published_at }} - {% endif %} + {% endif %} @@ -612,6 +679,12 @@ {{log.source_url }} {{ log.software_version }} + {% empty %} + + + There are no relevant records. + + {% endfor %} @@ -639,4 +712,4 @@ -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/vulnerabilities/templatetags/__init__.py b/vulnerabilities/templatetags/__init__.py index bdac1cd30..20854f2ad 100644 --- a/vulnerabilities/templatetags/__init__.py +++ b/vulnerabilities/templatetags/__init__.py @@ -3,6 +3,6 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/templatetags/url_filters.py b/vulnerabilities/templatetags/url_filters.py new file mode 100644 index 000000000..a6dda1dd8 --- /dev/null +++ b/vulnerabilities/templatetags/url_filters.py @@ -0,0 +1,11 @@ +from urllib.parse import quote + +import packageurl +from django import template + +register = template.Library() + + +@register.filter(name="url_quote") +def url_quote_filter(value): + return quote(str(value)) diff --git a/vulnerabilities/tests/__init__.py b/vulnerabilities/tests/__init__.py index bdac1cd30..20854f2ad 100644 --- a/vulnerabilities/tests/__init__.py +++ b/vulnerabilities/tests/__init__.py @@ -3,6 +3,6 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index f9216c742..de75014fb 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/example_importer_improver.py b/vulnerabilities/tests/example_importer_improver.py index bb74e3a3f..bcaa87c45 100644 --- a/vulnerabilities/tests/example_importer_improver.py +++ b/vulnerabilities/tests/example_importer_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/__init__.py b/vulnerabilities/tests/pipelines/__init__.py new file mode 100644 index 000000000..451fd4447 --- /dev/null +++ b/vulnerabilities/tests/pipelines/__init__.py @@ -0,0 +1,20 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import io + + +class TestLogger: + buffer = io.StringIO() + + def write(self, msg, level=None): + self.buffer.write(msg) + + def getvalue(self): + return self.buffer.getvalue() diff --git a/vulnerabilities/tests/pipelines/test_base_pipeline.py b/vulnerabilities/tests/pipelines/test_base_pipeline.py new file mode 100644 index 000000000..02e8c6b09 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_base_pipeline.py @@ -0,0 +1,125 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from unittest.mock import patch + +from django.test import TestCase +from django.utils import timezone +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities import models +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.tests.pipelines import TestLogger + +advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="dummy"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", +) + + +def get_advisory1(created_by="test_pipeline"): + return models.Advisory.objects.create( + aliases=advisory_data1.aliases, + summary=advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in advisory_data1.affected_packages], + references=[ref.to_dict() for ref in advisory_data1.references], + url=advisory_data1.url, + created_by=created_by, + date_collected=timezone.now(), + ) + + +class TestVulnerableCodePipeline(TestCase): + def test_on_failure(self): + class TestPipeline(VulnerableCodePipeline): + def __init__(self, test_logger): + super().__init__() + self.log = test_logger.write + + @classmethod + def steps(cls): + return (cls.step1,) + + def step1(self): + raise Exception("Something went wrong!") + + def on_failure(self): + self.log("Doing cleanup.") + + logger = TestLogger() + pipeline = TestPipeline(test_logger=logger) + + pipeline.execute() + log_result = logger.getvalue() + + self.assertIn("Pipeline failed", log_result) + self.assertIn("Running [on_failure] tasks", log_result) + + +class TestVulnerableCodeBaseImporterPipeline(TestCase): + @patch.object( + VulnerableCodeBaseImporterPipeline, + "collect_advisories", + return_value=[advisory_data1], + ) + @patch.object( + VulnerableCodeBaseImporterPipeline, + "advisories_count", + return_value=1, + ) + def test_collect_and_store_advisories(self, mock_advisories_count, mock_collect_advisories): + self.assertEqual(0, models.Advisory.objects.count()) + + base_pipeline = VulnerableCodeBaseImporterPipeline() + base_pipeline.pipeline_id = "test_pipeline" + + base_pipeline.collect_and_store_advisories() + + mock_advisories_count.assert_called_once() + mock_collect_advisories.assert_called_once() + + self.assertEqual(1, models.Advisory.objects.count()) + + collected_advisory = models.Advisory.objects.first() + result_aliases = collected_advisory.aliases + expected_aliases = advisory_data1.aliases + + self.assertEqual(expected_aliases, result_aliases) + self.assertEqual(base_pipeline.pipeline_id, collected_advisory.created_by) + + def test_import_new_advisories(self): + self.assertEqual(0, models.Vulnerability.objects.count()) + + base_pipeline = VulnerableCodeBaseImporterPipeline() + base_pipeline.pipeline_id = "test_pipeline" + advisory1 = get_advisory1() + base_pipeline.import_new_advisories() + + self.assertEqual(1, models.Vulnerability.objects.count()) + + imported_vulnerability = models.Vulnerability.objects.first() + + self.assertEqual(1, imported_vulnerability.aliases.count()) + + expected_alias = imported_vulnerability.aliases.first() + self.assertEqual(advisory1.aliases[0], expected_alias.alias) diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py new file mode 100644 index 000000000..a366d32e8 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -0,0 +1,34 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from decimal import Decimal + +import pytest + +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import Package +from vulnerabilities.pipelines.compute_package_risk import ComputePackageRiskPipeline +from vulnerabilities.tests.test_risk import vulnerability + + +@pytest.mark.django_db +def test_simple_risk_pipeline(vulnerability): + pkg = Package.objects.create(type="pypi", name="foo", version="2.3.0") + assert Package.objects.count() == 1 + + improver = ComputePackageRiskPipeline() + improver.execute() + + assert pkg.risk_score is None + + AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) + improver = ComputePackageRiskPipeline() + improver.execute() + + pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") + assert pkg.risk_score == Decimal("3.1") # max( 6.9 * 9/10 , 6.5 * 9/10 ) * .5 = 3.105 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py new file mode 100644 index 000000000..f54dad55d --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py @@ -0,0 +1,47 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_exploitdb import ExploitDBImproverPipeline + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_exploit_db_improver(mock_get): + mock_response = Mock(status_code=200) + with open(TEST_DATA, "r") as f: + mock_response.text = f.read() + mock_get.return_value = mock_response + + improver = ExploitDBImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + v1.save() + + Alias.objects.create(alias="CVE-2009-3699", vulnerability=v1) + + # Run Exploit-DB Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/test_kev_improver.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py similarity index 50% rename from vulnerabilities/tests/test_kev_improver.py rename to vulnerabilities/tests/pipelines/test_enhance_with_kev.py index d0b1c981a..a93c16555 100644 --- a/vulnerabilities/tests/test_kev_improver.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py @@ -1,41 +1,41 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os -from datetime import datetime from unittest import mock from unittest.mock import Mock import pytest -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improvers.vulnerability_kev import VulnerabilityKevImprover from vulnerabilities.models import Alias -from vulnerabilities.models import Kev +from vulnerabilities.models import Exploit from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_kev import VulnerabilityKevPipeline from vulnerabilities.utils import load_json BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "kev_data.json") +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") @pytest.mark.django_db @mock.patch("requests.get") def test_kev_improver(mock_get): - advisory_data = AdvisoryData( - aliases=["CVE-2022-21831"], - summary="Possible code injection vulnerability in Rails / Active Storage", - affected_packages=[], - references=[], - date_published=datetime.now(), - ) # to just run the improver - mock_response = Mock(status_code=200) mock_response.json.return_value = load_json(TEST_DATA) mock_get.return_value = mock_response - improver = VulnerabilityKevImprover() + improver = VulnerabilityKevPipeline() # Run the improver when there is no matching aliases - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 0 + improver.execute() + + assert Exploit.objects.count() == 0 v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") v1.save() @@ -43,5 +43,5 @@ def test_kev_improver(mock_get): Alias.objects.create(alias="CVE-2021-38647", vulnerability=v1) # Run Kev Improver again when there are matching aliases. - improver.get_inferences(advisory_data=advisory_data) - assert Kev.objects.count() == 1 + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py new file mode 100644 index 000000000..eea99e0ca --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py @@ -0,0 +1,44 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from unittest import mock +from unittest.mock import Mock + +import pytest + +from vulnerabilities.models import Alias +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines.enhance_with_metasploit import MetasploitImproverPipeline +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") + + +@pytest.mark.django_db +@mock.patch("requests.get") +def test_metasploit_improver(mock_get): + mock_response = Mock(status_code=200) + mock_response.json.return_value = load_json(TEST_DATA) + mock_get.return_value = mock_response + + improver = MetasploitImproverPipeline() + + # Run the improver when there is no matching aliases + improver.execute() + assert Exploit.objects.count() == 0 + + v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") + Alias.objects.create(alias="CVE-2007-4387", vulnerability=v1) + + # Run metasploit Improver again when there are matching aliases. + improver.execute() + assert Exploit.objects.count() == 1 diff --git a/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py b/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py new file mode 100644 index 000000000..192901c36 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py @@ -0,0 +1,71 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +from pathlib import Path +from unittest import mock + +from django.test import TestCase +from fetchcode.package_versions import PackageVersion +from packageurl import PackageURL + +from vulnerabilities.models import Package +from vulnerabilities.pipelines import flag_ghost_packages +from vulnerabilities.tests.pipelines import TestLogger + + +class FlagGhostPackagePipelineTest(TestCase): + data = Path(__file__).parent.parent / "test_data" + + @mock.patch("vulnerabilities.pipelines.flag_ghost_packages.versions") + def test_flag_ghost_package(self, mock_fetchcode_versions): + Package.objects.create(type="pypi", name="foo", version="2.3.0") + Package.objects.create(type="pypi", name="foo", version="3.0.0") + + mock_fetchcode_versions.return_value = [ + PackageVersion(value="2.3.0"), + ] + interesting_packages_qs = Package.objects.all() + base_purl = PackageURL(type="pypi", name="foo") + + self.assertEqual(0, Package.objects.filter(is_ghost=True).count()) + + flagged_package_count = flag_ghost_packages.flag_ghost_packages( + base_purl=base_purl, + packages=interesting_packages_qs, + ) + self.assertEqual(1, flagged_package_count) + self.assertEqual(1, Package.objects.filter(is_ghost=True).count()) + + @mock.patch("vulnerabilities.pipelines.flag_ghost_packages.versions") + def test_detect_and_flag_ghost_packages(self, mock_fetchcode_versions): + Package.objects.create(type="pypi", name="foo", version="2.3.0") + Package.objects.create(type="pypi", name="foo", version="3.0.0") + Package.objects.create( + type="deb", + namespace="debian", + name="foo", + version="3.0.0", + qualifiers={"distro": "trixie"}, + ) + + mock_fetchcode_versions.return_value = [ + PackageVersion(value="2.3.0"), + ] + + self.assertEqual(3, Package.objects.count()) + self.assertEqual(0, Package.objects.filter(is_ghost=True).count()) + + logger = TestLogger() + + flag_ghost_packages.detect_and_flag_ghost_packages(logger=logger.write) + expected = "Successfully flagged 1 ghost Packages" + + self.assertIn(expected, logger.getvalue()) + self.assertEqual(1, Package.objects.filter(is_ghost=True).count()) diff --git a/vulnerabilities/tests/test_github.py b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py similarity index 82% rename from vulnerabilities/tests/test_github.py rename to vulnerabilities/tests/pipelines/test_github_importer_pipeline.py index 2b5593137..dfa5ea371 100644 --- a/vulnerabilities/tests/test_github.py +++ b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py @@ -3,13 +3,14 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json import os from datetime import datetime +from pathlib import Path from unittest import mock import pytest @@ -23,23 +24,22 @@ from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers.github import GitHubAPIImporter -from vulnerabilities.importers.github import get_cwes_from_github_advisory -from vulnerabilities.importers.github import process_response from vulnerabilities.improvers.valid_versions import GitHubBasicImprover +from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline +from vulnerabilities.pipelines.github_importer import get_cwes_from_github_advisory +from vulnerabilities.pipelines.github_importer import process_response +from vulnerabilities.tests.pipelines import TestLogger from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN -from vulnerabilities.utils import GitHubTokenError -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "github_api") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "github_api" @pytest.mark.parametrize( "pkg_type", ["maven", "nuget", "gem", "golang", "composer", "pypi", "npm", "cargo"] ) def test_process_response_github_importer(pkg_type, regen=REGEN): - response_file = os.path.join(TEST_DATA, f"{pkg_type}.json") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") + response_file = TEST_DATA / f"{pkg_type}.json" + expected_file = TEST_DATA / f"{pkg_type}-expected.json" with open(response_file) as f: response = json.load(f) @@ -56,34 +56,49 @@ def test_process_response_github_importer(pkg_type, regen=REGEN): assert result == expected -def test_process_response_with_empty_vulnaribilities(caplog): - list(process_response({"data": {"securityVulnerabilities": {"edges": []}}}, "maven")) - assert "No vulnerabilities found for package_type: 'maven'" in caplog.text +def test_process_response_with_empty_vulnaribilities(): + logger = TestLogger() + list( + process_response( + {"data": {"securityVulnerabilities": {"edges": []}}}, + "maven", + logger=logger.write, + ) + ) + assert "No vulnerabilities found for package_type: 'maven'" in logger.getvalue() -def test_process_response_with_empty_vulnaribilities_2(caplog): +def test_process_response_with_empty_vulnaribilities_2(): + logger = TestLogger() list( process_response( - {"data": {"securityVulnerabilities": {"edges": [{"node": {}}, None]}}}, "maven" + {"data": {"securityVulnerabilities": {"edges": [{"node": {}}, None]}}}, + "maven", + logger=logger.write, ) ) - assert "No node found" in caplog.text + assert "No node found" in logger.getvalue() def test_github_importer_with_missing_credentials(): - with pytest.raises(GitHubTokenError) as e: - with mock.patch.dict(os.environ, {}, clear=True): - importer = GitHubAPIImporter() - list(importer.advisory_data()) + with mock.patch.dict(os.environ, {}, clear=True): + github_pipeline = GitHubAPIImporterPipeline() + status, error = github_pipeline.execute() + assert 1 == status + assert ( + "Cannot call GitHub API without a token set in the GH_TOKEN environment variable." + in error + ) @mock.patch("vulnerabilities.utils._get_gh_response") def test_github_importer_with_missing_credentials_2(mock_response): mock_response.return_value = {"message": "Bad credentials"} - with pytest.raises(GitHubTokenError) as e: - with mock.patch.dict(os.environ, {"GH_TOKEN": "FOOD"}, clear=True): - importer = GitHubAPIImporter() - list(importer.advisory_data()) + with mock.patch.dict(os.environ, {"GH_TOKEN": "FOOD"}, clear=True): + github_pipeline = GitHubAPIImporterPipeline() + status, error = github_pipeline.execute() + assert 1 == status + assert "Invalid GitHub token: Bad credentials" in error def valid_versions(): @@ -283,11 +298,18 @@ def test_github_improver(mock_response, regen=REGEN): @mock.patch("fetchcode.package_versions.get_response") def test_get_package_versions(mock_response): - with open(os.path.join(BASE_DIR, "test_data", "package_manager_data", "pypi.json"), "r") as f: + with open(TEST_DATA.parent / "package_manager_data" / "pypi.json", "r") as f: mock_response.return_value = json.load(f) improver = GitHubBasicImprover() valid_versions = [ + "1.0.1", + "1.0.2", + "1.0.3", + "1.0.4", + "1.1", + "1.1.1", + "1.1.2", "1.1.3", "1.1.4", "1.10", @@ -302,7 +324,9 @@ def test_get_package_versions(mock_response): "1.10a1", "1.10b1", "1.10rc1", + "vulnerabilities", ] + result = sorted( improver.get_package_versions(package_url=PackageURL(type="pypi", name="django")) ) diff --git a/vulnerabilities/tests/test_gitlab.py b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py similarity index 64% rename from vulnerabilities/tests/test_gitlab.py rename to vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py index bc2bfcaea..c3dc7be43 100644 --- a/vulnerabilities/tests/test_gitlab.py +++ b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py @@ -3,32 +3,39 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json -import os from pathlib import Path from unittest import mock import pytest from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.gitlab import parse_gitlab_advisory from vulnerabilities.improvers.default import DefaultImprover from vulnerabilities.improvers.valid_versions import GitLabBasicImprover +from vulnerabilities.pipelines import gitlab_importer from vulnerabilities.tests import util_tests +from vulnerabilities.tests.pipelines import TestLogger -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "gitlab") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "gitlab" @pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) def test_parse_yaml_file(pkg_type): - response_file = os.path.join(TEST_DATA, f"{pkg_type}.yaml") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") - advisory = parse_gitlab_advisory(Path(response_file), Path(response_file).parent) + response_file = TEST_DATA / f"{pkg_type}.yaml" + expected_file = TEST_DATA / f"{pkg_type}-expected.json" + test_pipeline = gitlab_importer.GitLabImporterPipeline() + logger = TestLogger() + advisory = gitlab_importer.parse_gitlab_advisory( + response_file, + response_file.parent, + test_pipeline.gitlab_scheme_by_purl_type, + test_pipeline.purl_type_by_gitlab_scheme, + logger.write, + ) util_tests.check_results_against_json(advisory.to_dict(), expected_file) @@ -45,27 +52,11 @@ def valid_versions(pkg_type): "9.1.6", "10.0.0", ], - "gem": [ - "4.2.0.beta1", - "4.2.0.beta2", - "4.2.0.beta3", - ], - "golang": [ - "3.7.0", - "3.7.1", - ], + "gem": ["4.2.0.beta1", "4.2.0.beta2", "4.2.0.beta3"], + "golang": ["3.7.0", "3.7.1"], "nuget": ["1.11.0", "1.11.1", "1.11.2", "1.09.1"], - "npm": [ - "2.14.2", - "2.13.2", - "2.11.2", - ], - "pypi": [ - "1.0", - "0.9", - "0.8", - "1.1", - ], + "npm": ["2.14.2", "2.13.2", "2.11.2"], + "pypi": ["1.0", "0.9", "0.8", "1.1"], "composer": [], } return valid_versions_by_package_type[pkg_type] @@ -74,9 +65,9 @@ def valid_versions(pkg_type): @mock.patch("vulnerabilities.improvers.valid_versions.GitLabBasicImprover.get_package_versions") @pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) def test_gitlab_improver(mock_response, pkg_type): - advisory_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json") - expected_file = os.path.join(TEST_DATA, f"{pkg_type}-improver-expected.json") - with open(advisory_file) as exp: + advisory_file = TEST_DATA / f"{pkg_type}-expected.json" + expected_file = TEST_DATA / f"{pkg_type}-improver-expected.json" + with advisory_file.open() as exp: advisory = AdvisoryData.from_dict(json.load(exp)) mock_response.return_value = list(valid_versions(pkg_type)) improvers = [GitLabBasicImprover(), DefaultImprover()] diff --git a/vulnerabilities/tests/test_nginx.py b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py similarity index 82% rename from vulnerabilities/tests/test_nginx.py rename to vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py index c27ef2d10..c4bce99a6 100644 --- a/vulnerabilities/tests/test_nginx.py +++ b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -15,16 +15,16 @@ from bs4 import BeautifulSoup from commoncode import testcase from django.db.models.query import QuerySet +from univers.version_range import NginxVersionRange from vulnerabilities import models from vulnerabilities import severity_systems -from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import Reference from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers import nginx from vulnerabilities.improvers.valid_versions import NginxBasicImprover from vulnerabilities.models import Advisory +from vulnerabilities.pipelines import nginx_importer from vulnerabilities.tests import util_tests from vulnerabilities.utils import is_vulnerable_nginx_version @@ -40,14 +40,14 @@ class TestNginxImporterAndImprover(testcase.FileBasedTesting): - test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "nginx") + test_data_dir = Path(__file__).parent.parent / "test_data" / "nginx" def test_is_vulnerable(self): # Not vulnerable: 1.17.3+, 1.16.1+ # Vulnerable: 1.9.5-1.17.2 - vcls = nginx.NginxVersionRange.version_class - affected_version_range = nginx.NginxVersionRange.from_native("1.9.5-1.17.2") + vcls = NginxVersionRange.version_class + affected_version_range = NginxVersionRange.from_native("1.9.5-1.17.2") fixed_versions = [vcls("1.17.3"), vcls("1.16.1")] version = vcls("1.9.4") @@ -133,10 +133,10 @@ def test_parse_advisory_data_from_paragraph(self): ], } - result = nginx.parse_advisory_data_from_paragraph(vuln_info) + result = nginx_importer.parse_advisory_data_from_paragraph(vuln_info) assert result.to_dict() == expected - def test_advisory_data_from_text(self): + def test_collect_advisories(self): test_file = self.get_test_loc("security_advisories.html") with open(test_file) as tf: test_text = tf.read() @@ -145,52 +145,49 @@ def test_advisory_data_from_text(self): "security_advisories-advisory_data-expected.json", must_exist=False ) - results = [na.to_dict() for na in nginx.advisory_data_from_text(test_text)] + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text + results = [na.to_dict() for na in test_pipeline.collect_advisories()] util_tests.check_results_against_json(results, expected_file) @pytest.mark.django_db(transaction=True) - def test_NginxImporter(self): + def test_NginxImporterPipeline_collect_and_store_advisories(self): + test_file = self.get_test_loc("security_advisories.html") + with open(test_file) as tf: + test_text = tf.read() + + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text expected_file = self.get_test_loc( "security_advisories-importer-expected.json", must_exist=False ) - results, _cls = self.run_import() - util_tests.check_results_against_json(results, expected_file) + test_pipeline.collect_and_store_advisories() - # run again as there should be no duplicates - results, _cls = self.run_import() + results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) util_tests.check_results_against_json(results, expected_file) - def run_import(self): - """ - Return a list of imported Advisory model objects and the MockImporter - used. - """ - - class MockImporter(nginx.NginxImporter): - """ - A mocked NginxImporter that loads content from a file rather than - making a network call. - """ - - def fetch(self): - with open(test_file) as tf: - return tf.read() - - test_file = self.get_test_loc("security_advisories.html") + # run again as there should be no duplicates + test_pipeline.collect_and_store_advisories() - ImportRunner(MockImporter).run() - return list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)), MockImporter + results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) + util_tests.check_results_against_json(results, expected_file) @pytest.mark.django_db(transaction=True) def test_NginxBasicImprover__interesting_advisories(self): - advisories, importer_class = self.run_import() + test_file = self.get_test_loc("security_advisories.html") + with open(test_file) as tf: + test_text = tf.read() + + test_pipeline = nginx_importer.NginxImporterPipeline() + test_pipeline.advisory_data = test_text + advisories = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) class MockNginxBasicImprover(NginxBasicImprover): @property def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=importer_class.qualified_name) + return Advisory.objects.filter(created_by=test_pipeline.pipeline_id) improver = MockNginxBasicImprover() interesting_advisories = list( @@ -198,7 +195,7 @@ def interesting_advisories(self) -> QuerySet: ) assert interesting_advisories == advisories - @mock.patch("fetchcode.package_versions.github_response") + @mock.patch("fetchcode.utils.github_response") def test_NginxBasicImprover_fetch_nginx_version_from_git_tags(self, mock_fetcher): reponse_files = [ "github-nginx-nginx-0.json", diff --git a/vulnerabilities/tests/test_npm.py b/vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py similarity index 86% rename from vulnerabilities/tests/test_npm.py rename to vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py index 28ca7a548..bcfb83f62 100644 --- a/vulnerabilities/tests/test_npm.py +++ b/vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py @@ -1,15 +1,17 @@ -# Author: Navonil Das (@NavonilDas) # # Copyright (c) nexB Inc. and others. All rights reserved. # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +# Author: Navonil Das (@NavonilDas) + import json import os +from pathlib import Path from unittest.mock import patch from packageurl import PackageURL @@ -19,19 +21,18 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importers.npm import NpmImporter from vulnerabilities.improvers.default import DefaultImprover from vulnerabilities.improvers.valid_versions import NpmImprover +from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline from vulnerabilities.tests import util_tests from vulnerabilities.utils import load_json -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") +TEST_DATA = data = Path(__file__).parent.parent / "test_data" / "npm" def test_npm_importer(): file = os.path.join(TEST_DATA, "npm_sample.json") - result = [adv.to_dict() for adv in NpmImporter().to_advisory_data(file=file)] + result = [adv.to_dict() for adv in NpmImporterPipeline().to_advisory_data(file=file)] expected_file = os.path.join(TEST_DATA, f"parse-advisory-npm-expected.json") util_tests.check_results_against_json(result, expected_file) @@ -47,7 +48,7 @@ def test_get_affected_package(): constraints=(VersionConstraint(comparator="<", version=SemverVersion(string="1.3.3")),) ), fixed_version=SemverVersion(string="1.3.3"), - ) == NpmImporter().get_affected_package(data, "npm") + ) == NpmImporterPipeline().get_affected_package(data, "npm") @patch("vulnerabilities.improvers.valid_versions.NpmImprover.get_package_versions") diff --git a/vulnerabilities/tests/test_nvd.py b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py similarity index 83% rename from vulnerabilities/tests/test_nvd.py rename to vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py index 702faa7f4..5b90ca986 100644 --- a/vulnerabilities/tests/test_nvd.py +++ b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py @@ -3,19 +3,17 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json -import os +from pathlib import Path -from vulnerabilities.importers import nvd +from vulnerabilities.pipelines import nvd_importer from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/nvd/nvd_test.json") -REJECTED_CVE = os.path.join(BASE_DIR, "test_data/nvd/rejected_nvd.json") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "nvd" def load_test_data(file): @@ -37,10 +35,11 @@ def sorted_advisory_data(advisory_data): def test_to_advisories_skips_hardware(regen=REGEN): - expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-expected.json") + expected_file = TEST_DATA / "nvd-expected.json" - test_data = load_test_data(file=TEST_DATA) - result = [data.to_dict() for data in nvd.to_advisories(test_data)] + test_file = TEST_DATA / "nvd_test.json" + test_data = load_test_data(file=test_file) + result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] result = sorted_advisory_data(result) if regen: @@ -56,10 +55,11 @@ def test_to_advisories_skips_hardware(regen=REGEN): def test_to_advisories_marks_rejected_cve(regen=REGEN): - expected_file = os.path.join(BASE_DIR, "test_data/nvd/nvd-rejected-expected.json") + expected_file = TEST_DATA / "nvd-rejected-expected.json" - test_data = load_test_data(file=REJECTED_CVE) - result = [data.to_dict() for data in nvd.to_advisories(test_data)] + test_file = TEST_DATA / "rejected_nvd.json" + test_data = load_test_data(file=test_file) + result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] result = sorted_advisory_data(result) if regen: @@ -168,14 +168,16 @@ def test_CveItem_cpes(): "cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*", ] - found_cpes = nvd.CveItem(cve_item=get_test_cve_item()).cpes + found_cpes = nvd_importer.CveItem(cve_item=get_test_cve_item()).cpes assert found_cpes == expected_cpes def test_is_related_to_hardware(): - assert nvd.is_related_to_hardware("cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*") - assert not nvd.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*") - assert not nvd.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*") + assert nvd_importer.is_related_to_hardware("cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*") + assert not nvd_importer.is_related_to_hardware( + "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*" + ) + assert not nvd_importer.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*") def test_CveItem_summary_with_single_summary(): @@ -186,7 +188,7 @@ def test_CveItem_summary_with_single_summary(): "be allocated than expected." ) - assert nvd.CveItem(cve_item=get_test_cve_item()).summary == expected_summary + assert nvd_importer.CveItem(cve_item=get_test_cve_item()).summary == expected_summary def test_CveItem_reference_urls(): @@ -195,4 +197,4 @@ def test_CveItem_reference_urls(): "http://kqueue.org/blog/2012/03/05/memory-allocator-security-revisited/", ] - assert nvd.CveItem(cve_item=get_test_cve_item()).reference_urls == expected_urls + assert nvd_importer.CveItem(cve_item=get_test_cve_item()).reference_urls == expected_urls diff --git a/vulnerabilities/tests/pipelines/test_pipeline_id.py b/vulnerabilities/tests/pipelines/test_pipeline_id.py new file mode 100644 index 000000000..6eb1e1008 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_pipeline_id.py @@ -0,0 +1,66 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import importlib +import inspect +import unittest +from pathlib import Path + +from vulnerabilities.pipelines import VulnerableCodePipeline + +PIPELINE_DIR = Path(__file__).parent.parent.parent / "pipelines" + + +class PipelineTests(unittest.TestCase): + def setUp(self): + self.pipeline_dict = self.collect_pipeline_ids() + + def collect_pipeline_ids(self): + """Return pipeline_ids from all the VulnerableCodePipeline.""" + pipeline_dict = {} + + for pipeline in PIPELINE_DIR.glob("*.py"): + if pipeline.name == "__init__.py": + continue + + module_name = pipeline.stem + module = importlib.import_module(f"vulnerabilities.pipelines.{module_name}") + + for _, obj in inspect.getmembers(module, inspect.isclass): + if issubclass(obj, VulnerableCodePipeline) and obj is not VulnerableCodePipeline: + pipeline_id = obj.pipeline_id + pipeline_dict[obj] = pipeline_id + break + + return pipeline_dict + + def test_no_empty_pipeline_ids(self): + empty_pipeline_ids = [cls for cls, pid in self.pipeline_dict.items() if pid == ""] + + if empty_pipeline_ids: + error_messages = [ + f"{cls.__name__} has empty pipeline_id." for cls in empty_pipeline_ids + ] + error_message = "`pipeline_id` should not be empty string:\n" + "\n".join( + error_messages + ) + assert False, error_message + + def test_no_none_pipeline_ids(self): + none_pipeline_ids = [cls for cls, pid in self.pipeline_dict.items() if pid == None] + + if none_pipeline_ids: + error_messages = [f"{cls.__name__} has None pipeline_id." for cls in none_pipeline_ids] + error_message = "`pipeline_id` should not be None:\n" + "\n".join(error_messages) + assert False, error_message + + def test_unique_pipeline_ids(self): + pipeline_ids = self.pipeline_dict.values() + unique_ids = set(pipeline_ids) + assert len(pipeline_ids) == len(unique_ids), "`pipeline_id` should be unique." diff --git a/vulnerabilities/tests/test_pypa.py b/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py similarity index 72% rename from vulnerabilities/tests/test_pypa.py rename to vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py index 1a59260e6..0bb631012 100644 --- a/vulnerabilities/tests/test_pypa.py +++ b/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py @@ -3,10 +3,12 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + import os +from pathlib import Path from unittest import TestCase import saneyaml @@ -14,14 +16,14 @@ from vulnerabilities.importers.osv import parse_advisory_data from vulnerabilities.tests import util_tests -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/pypa") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "pypa" -class TestPyPaImporter(TestCase): +class TestPyPaImporterPipeline(TestCase): def test_to_advisories_with_summary(self): - with open(os.path.join(TEST_DATA, "pypa_test.yaml")) as f: - mock_response = saneyaml.load(f) + pypa_advisory_path = TEST_DATA / "pypa_test.yaml" + + mock_response = saneyaml.load(pypa_advisory_path.read_text()) expected_file = os.path.join(TEST_DATA, "pypa-expected.json") imported_data = parse_advisory_data( mock_response, diff --git a/vulnerabilities/tests/test_pysec.py b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py similarity index 69% rename from vulnerabilities/tests/test_pysec.py rename to vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py index dcba3a776..c38256c01 100644 --- a/vulnerabilities/tests/test_pysec.py +++ b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py @@ -3,28 +3,28 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + import json -import os +from pathlib import Path from unittest import TestCase from vulnerabilities.importers.osv import parse_advisory_data from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN from vulnerabilities.tests.util_tests import check_results_against_json -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "pysec" class TestPyPIImporter(TestCase): def test_to_advisories_with_summary(self): - with open(os.path.join(TEST_DATA, "pysec-advisories_with_summary.json")) as f: + with open(TEST_DATA / "pysec-advisories_with_summary.json") as f: mock_response = json.load(f) results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_summary-expected.json") + expected_file = TEST_DATA / "pysec-advisories_with_summary-expected.json" check_results_against_json( results=results, expected_file=expected_file, @@ -32,12 +32,12 @@ def test_to_advisories_with_summary(self): ) def test_to_advisories_without_summary(self): - with open(os.path.join(TEST_DATA, "pysec-advisories_without_summary.json")) as f: + with open(TEST_DATA / "pysec-advisories_without_summary.json") as f: mock_response = json.load(f) results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_without_summary-expected.json") + expected_file = TEST_DATA / "pysec-advisories_without_summary-expected.json" check_results_against_json( results=results, expected_file=expected_file, @@ -45,14 +45,14 @@ def test_to_advisories_without_summary(self): ) def test_to_advisories_with_cwe(self): - with open(os.path.join(TEST_DATA, "pysec-advisory_with_cwe.json")) as f: + with open(TEST_DATA / "pysec-advisory_with_cwe.json") as f: mock_response = json.load(f) results = parse_advisory_data( raw_data=mock_response, supported_ecosystems=["pypi"], advisory_url="https://tes.com" ).to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_cwe-expected.json") + expected_file = TEST_DATA / "pysec-advisories_with_cwe-expected.json" check_results_against_json( results=results, expected_file=expected_file, diff --git a/vulnerabilities/tests/pipes/test_advisory.py b/vulnerabilities/tests/pipes/test_advisory.py new file mode 100644 index 000000000..a371ca551 --- /dev/null +++ b/vulnerabilities/tests/pipes/test_advisory.py @@ -0,0 +1,75 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest +from django.utils import timezone +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities import models +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference +from vulnerabilities.pipes.advisory import import_advisory + +advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="dummy"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", +) + + +def get_advisory1(created_by="test_pipeline"): + return models.Advisory.objects.create( + aliases=advisory_data1.aliases, + summary=advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in advisory_data1.affected_packages], + references=[ref.to_dict() for ref in advisory_data1.references], + url=advisory_data1.url, + created_by=created_by, + date_collected=timezone.now(), + ) + + +def get_all_vulnerability_relationships_objects(): + return { + "vulnerabilities": list(models.Vulnerability.objects.all()), + "aliases": list(models.Alias.objects.all()), + "references": list(models.VulnerabilityReference.objects.all()), + "advisories": list(models.Advisory.objects.all()), + "packages": list(models.Package.objects.all()), + "references": list(models.VulnerabilityReference.objects.all()), + "severity": list(models.VulnerabilitySeverity.objects.all()), + } + + +@pytest.mark.django_db +def test_vulnerability_pipes_importer_import_advisory(): + advisory1 = get_advisory1(created_by="test_importer_pipeline") + import_advisory(advisory=advisory1, pipeline_id="test_importer_pipeline") + all_vulnerability_relation_objects = get_all_vulnerability_relationships_objects() + import_advisory(advisory=advisory1, pipeline_id="test_importer_pipeline") + assert all_vulnerability_relation_objects == get_all_vulnerability_relationships_objects() + + +@pytest.mark.django_db +def test_vulnerability_pipes_importer_import_advisory_different_pipelines(): + advisory1 = get_advisory1(created_by="test_importer_pipeline") + import_advisory(advisory=advisory1, pipeline_id="test_importer1_pipeline") + all_vulnerability_relation_objects = get_all_vulnerability_relationships_objects() + import_advisory(advisory=advisory1, pipeline_id="test_importer2_pipeline") + assert all_vulnerability_relation_objects == get_all_vulnerability_relationships_objects() diff --git a/vulnerabilities/tests/test_affected_package.py b/vulnerabilities/tests/test_affected_package.py index 2ebbcbddb..f56551d65 100644 --- a/vulnerabilities/tests/test_affected_package.py +++ b/vulnerabilities/tests/test_affected_package.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_alpine.py b/vulnerabilities/tests/test_alpine.py index d65e27401..1ab74a89a 100644 --- a/vulnerabilities/tests/test_alpine.py +++ b/vulnerabilities/tests/test_alpine.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index a57437d01..853eafbd0 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py index 720196381..92c76c7b1 100644 --- a/vulnerabilities/tests/test_apache_kafka.py +++ b/vulnerabilities/tests/test_apache_kafka.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py index 3ed522597..51d33634f 100644 --- a/vulnerabilities/tests/test_apache_tomcat.py +++ b/vulnerabilities/tests/test_apache_tomcat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 33a71bb08..a5f80aa06 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -3,29 +3,27 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json import os -from collections import OrderedDict from urllib.parse import quote from django.test import TestCase from django.test import TransactionTestCase from django.test.client import RequestFactory -from packageurl import PackageURL from rest_framework import status from rest_framework.test import APIClient -from vulnerabilities.api import MinimalPackageSerializer from vulnerabilities.api import PackageSerializer from vulnerabilities.api import VulnerabilityReferenceSerializer +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias from vulnerabilities.models import ApiUser +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference @@ -35,6 +33,7 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data") +TEST_DIR = os.path.join(TEST_DATA, "api") def cleaned_response(response): @@ -212,8 +211,8 @@ def setUp(self): self.pkg1 = Package.objects.create(name="flask", type="pypi", version="0.1.2") self.pkg2 = Package.objects.create(name="flask", type="deb", version="0.1.2") for pkg in [self.pkg1, self.pkg2]: - PackageRelatedVulnerability.objects.create( - package=pkg, vulnerability=self.vulnerability, fix=True + FixingPackageRelatedVulnerability.objects.create( + package=pkg, vulnerability=self.vulnerability ) self.reference1 = VulnerabilityReference.objects.create( @@ -221,8 +220,8 @@ def setUp(self): url="https://.com", ) - VulnerabilitySeverity.objects.create( - reference=self.reference1, + severity = VulnerabilitySeverity.objects.create( + url="https://.com", scoring_system=EPSS.identifier, scoring_elements=".0016", value="0.526", @@ -238,6 +237,7 @@ def setUp(self): cwe_id=10000 ) # cwe not present in weaknesses_db self.invalid_weaknesses.vulnerabilities.add(self.vulnerability) + self.vulnerability.severities.add(severity) def test_api_status(self): response = self.csrf_client.get("/api/vulnerabilities/") @@ -256,6 +256,7 @@ def test_api_with_single_vulnerability(self): "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", "vulnerability_id": self.vulnerability.vulnerability_id, "summary": "test", + "severity_range_score": None, "aliases": [], "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", "fixed_packages": [ @@ -294,9 +295,13 @@ def test_api_with_single_vulnerability(self): { "cwe_id": 119, "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", - "description": "The software performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", + "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } def test_api_with_single_vulnerability_with_filters(self): @@ -307,6 +312,7 @@ def test_api_with_single_vulnerability_with_filters(self): "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", "vulnerability_id": self.vulnerability.vulnerability_id, "summary": "test", + "severity_range_score": None, "aliases": [], "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", "fixed_packages": [ @@ -338,11 +344,64 @@ def test_api_with_single_vulnerability_with_filters(self): { "cwe_id": 119, "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", - "description": "The software performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", + "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", + }, + ], + "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + } + + def test_api_with_single_vulnerability_no_ghost_fix(self): + self.pkg2.is_ghost = True + self.pkg1.is_ghost = True + self.pkg2.save() + self.pkg1.save() + + response = self.csrf_client.get( + f"/api/vulnerabilities/{self.vulnerability.id}", format="json" + ).data + + expected = { + "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", + "vulnerability_id": self.vulnerability.vulnerability_id, + "summary": "test", + "severity_range_score": None, + "aliases": [], + "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", + "fixed_packages": [], + "affected_packages": [], + "references": [ + { + "reference_url": "https://.com", + "reference_id": "", + "reference_type": "", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://.com", + } + ], + "weaknesses": [ + { + "cwe_id": 119, + "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", + "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", }, ], + "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } + assert expected == response + def set_as_affected_by(package, vulnerability): """ @@ -362,11 +421,16 @@ def _set_pkg_as(package, vulnerability, fixing=False): """ Set the ``package`` Package as affected or fixing the ``vulnerability`` Vulnerability. """ - PackageRelatedVulnerability.objects.create( - package=package, - vulnerability=vulnerability, - fix=fixing, - ) + if fixing: + FixingPackageRelatedVulnerability.objects.create( + package=package, + vulnerability=vulnerability, + ) + else: + AffectedByPackageRelatedVulnerability.objects.create( + package=package, + vulnerability=vulnerability, + ) def create_vuln(vcid, aliases=()): @@ -386,7 +450,7 @@ def add_aliases(vuln, aliases): Alias.objects.create(alias=alias, vulnerability=vuln) -class APITestCasePackage(TestCase): +class APIPerformanceTest(TestCase): def setUp(self): self.user = ApiUser.objects.create_api_user(username="e@mail.com") self.auth = f"Token {self.user.auth_token.key}" @@ -425,6 +489,7 @@ def setUp(self): self.pkg_2_14_0_rc1 = from_purl( "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + self.pkg_2_12_6.calculate_version_rank set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) @@ -437,24 +502,147 @@ def setUp(self): set_as_affected_by(package=self.pkg_2_13_2, vulnerability=self.vul2) set_as_fixing(package=self.pkg_2_13_2, vulnerability=self.vul1) - def test_api_with_package_with_no_vulnerabilities(self): - affected_vulnerabilities = [] - vuln = { - "foo": "bar", - } + def test_api_packages_all_num_queries(self): + with self.assertNumQueries(4): + # There are 4 queries: + # 1. SAVEPOINT + # 2. Authenticating user + # 3. Get all vulnerable packages + # 4. RELEASE SAVEPOINT + response = self.csrf_client.get(f"/api/packages/all", format="json").data + + assert len(response) == 3 + assert list(response) == [ + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1", + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + ] + + def test_api_packages_single_num_queries(self): + with self.assertNumQueries(8): + self.csrf_client.get(f"/api/packages/{self.pkg_2_14_0_rc1.id}", format="json") + + def test_api_packages_single_with_purl_in_query_num_queries(self): + with self.assertNumQueries(9): + self.csrf_client.get(f"/api/packages/?purl={self.pkg_2_14_0_rc1.purl}", format="json") + + def test_api_packages_single_with_purl_no_version_in_query_num_queries(self): + with self.assertNumQueries(64): + self.csrf_client.get( + f"/api/packages/?purl=pkg:maven/com.fasterxml.jackson.core/jackson-databind", + format="json", + ) + + def test_api_packages_bulk_search(self): + with self.assertNumQueries(45): + packages = [self.pkg_2_12_6, self.pkg_2_12_6_1, self.pkg_2_13_1] + purls = [p.purl for p in packages] + + data = {"purls": purls, "purl_only": False, "plain_purl": True} - package_with_no_vulnerabilities = MinimalPackageSerializer.get_vulnerability( - self, - vuln, + resp = self.csrf_client.post( + f"/api/packages/bulk_search", + data=json.dumps(data), + content_type="application/json", + ).json() + + def test_api_packages_with_lookup(self): + with self.assertNumQueries(14): + data = {"purl": self.pkg_2_12_6.purl} + + resp = self.csrf_client.post( + f"/api/packages/lookup", + data=json.dumps(data), + content_type="application/json", + ).json() + + def test_api_packages_bulk_lookup(self): + with self.assertNumQueries(45): + packages = [self.pkg_2_12_6, self.pkg_2_12_6_1, self.pkg_2_13_1] + purls = [p.purl for p in packages] + + data = {"purls": purls} + + resp = self.csrf_client.post( + f"/api/packages/bulk_lookup", + data=json.dumps(data), + content_type="application/json", + ).json() + + +class APITestCasePackage(TestCase): + def setUp(self): + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.csrf_client = APIClient(enforce_csrf_checks=True) + self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) + + # This setup creates the following data: + # vulnerabilities: vul1, vul2, vul3 + # pkg:maven/com.fasterxml.jackson.core/jackson-databind + # with these versions: + # pkg_2_12_6: @ 2.12.6 affected by fixing vul3 + # pkg_2_12_6_1: @ 2.12.6.1 affected by vul2 fixing vul1 + # pkg_2_13_1: @ 2.13.1 affected by vul1 fixing vul3 + # pkg_2_13_2: @ 2.13.2 affected by vul2 fixing vul1 + # pkg_2_14_0_rc1: @ 2.14.0-rc1 affected by fixing + + # searched-for pkg's vuln + self.vul1 = create_vuln("VCID-vul1-vul1-vul1", ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"]) + self.vul2 = create_vuln("VCID-vul2-vul2-vul2") + # This is the vuln fixed by the searched-for pkg -- and by a lesser version (created below), + # which WILL be included in the API + self.vul3 = create_vuln("VCID-vul3-vul3-vul3", ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"]) + + from_purl = Package.objects.from_purl + # lesser-version pkg that also fixes the vuln fixed by the searched-for pkg + self.pkg_2_12_6 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6") + # this is a lesser version omitted from the API that fixes searched-for pkg's vuln + self.pkg_2_12_6_1 = from_purl( + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" + ) + # searched-for pkg + self.pkg_2_13_1 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1") + # this is a greater version that fixes searched-for pkg's vuln + self.pkg_2_13_2 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2") + # This addresses both next and latest non-vulnerable pkg + self.pkg_2_14_0_rc1 = from_purl( + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + self.pkg_2_12_6.calculate_version_rank + + self.ref = VulnerabilityReference.objects.create( + reference_type="advisory", reference_id="CVE-xxx-xxx", url="https://example.com" + ) + + self.severity = VulnerabilitySeverity.objects.create( + url="https://example.com", + scoring_system=EPSS.identifier, + scoring_elements=".0016", + value="0.526", + ) + self.vul1.references.add(self.ref) + self.vul1.severities.add(self.severity) + + self.vul3.references.add(self.ref) + self.vul3.severities.add(self.severity) + + set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) + + set_as_affected_by(package=self.pkg_2_12_6_1, vulnerability=self.vul2) + set_as_fixing(package=self.pkg_2_12_6_1, vulnerability=self.vul1) + + set_as_affected_by(package=self.pkg_2_13_1, vulnerability=self.vul1) + set_as_fixing(package=self.pkg_2_13_1, vulnerability=self.vul3) - assert package_with_no_vulnerabilities is None + set_as_affected_by(package=self.pkg_2_13_2, vulnerability=self.vul2) + set_as_fixing(package=self.pkg_2_13_2, vulnerability=self.vul1) def test_api_with_lesser_and_greater_fixed_by_packages(self): response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data - expected_response = { - "url": f"http://testserver/api/packages/{self.pkg_2_13_1.id}", + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", "type": "maven", "namespace": "com.fasterxml.jackson.core", @@ -466,109 +654,93 @@ def test_api_with_lesser_and_greater_fixed_by_packages(self): "next_non_vulnerable_version": "2.14.0-rc1", "latest_non_vulnerable_version": "2.14.0-rc1", "affected_by_vulnerabilities": [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/vulnerabilities/{self.vul1.id}", - ), - ("vulnerability_id", "VCID-vul1-vul1-vul1"), - ("summary", "This is VCID-vul1-vul1-vul1"), - ("references", []), - ( - "fixed_packages", - [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_13_2.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - ), - ("is_vulnerable", True), - ( - "affected_by_vulnerabilities", - [{"vulnerability": "VCID-vul2-vul2-vul2"}], - ), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - ), - ] - ) + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } ], - ), - ("aliases", ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"]), - ("resource_url", "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1"), - ] - ) + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } ], "fixing_vulnerabilities": [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/vulnerabilities/{self.vul3.id}", - ), - ("vulnerability_id", "VCID-vul3-vul3-vul3"), - ("summary", "This is VCID-vul3-vul3-vul3"), - ("references", []), - ( - "fixed_packages", - [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_12_6.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", - ), - ("is_vulnerable", False), - ("affected_by_vulnerabilities", []), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", - ), - ] - ), - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_13_1.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - ), - ("is_vulnerable", True), - ( - "affected_by_vulnerabilities", - [{"vulnerability": "VCID-vul1-vul1-vul1"}], - ), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - ), - ] - ), + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), + "vulnerability_id": "VCID-vul3-vul3-vul3", + "summary": "This is VCID-vul3-vul3-vul3", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } ], - ), - ("aliases", ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"]), - ("resource_url", "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3"), - ] - ) + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + "is_vulnerable": False, + "affected_by_vulnerabilities": [], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + }, + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul1-vul1-vul1"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + }, + ], + "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", + } ], + "risk_score": None, "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", } - assert json.dumps(response, indent=2) == json.dumps(expected_response, indent=2) + assert response == expected def test_is_vulnerable_attribute_only_exists_on_queryset(self): assert not hasattr(self.pkg_2_13_1, "is_vulnerable") @@ -603,7 +775,7 @@ def test_api_with_all_vulnerable_packages(self): response = self.csrf_client.get(f"/api/packages/all", format="json").data assert len(response) == 3 - assert response == [ + assert list(response) == [ "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1", "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", @@ -620,6 +792,176 @@ def test_api_with_ignorning_qualifiers(self): == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + def test_api_with_ghost_package_no_fixing_vulnerabilities(self): + self.pkg_2_13_1.is_ghost = True + self.pkg_2_13_1.save() + + response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data + + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.1", + "qualifiers": {}, + "subpath": "", + "is_vulnerable": True, + "next_non_vulnerable_version": "2.12.6", + "latest_non_vulnerable_version": "2.14.0-rc1", + "affected_by_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } + ], + "fixing_vulnerabilities": [], + "risk_score": None, + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + } + + assert response == expected + + def test_api_with_ghost_package_no_next_latest_non_vulnerabilities(self): + self.pkg_2_14_0_rc1.is_ghost = True + self.pkg_2_14_0_rc1.save() + + response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data + + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.1", + "qualifiers": {}, + "subpath": "", + "is_vulnerable": True, + "next_non_vulnerable_version": None, + "latest_non_vulnerable_version": None, + "affected_by_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } + ], + "fixing_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), + "vulnerability_id": "VCID-vul3-vul3-vul3", + "summary": "This is VCID-vul3-vul3-vul3", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + "is_vulnerable": False, + "affected_by_vulnerabilities": [], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + }, + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul1-vul1-vul1"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + }, + ], + "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", + } + ], + "risk_score": None, + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + } + + assert response == expected + class CPEApi(TestCase): def setUp(self): @@ -646,6 +988,46 @@ def test_api_response(self): self.assertEqual(response["count"], 1) +class TestCPEApiWithPackageVulnerabilityRelation(TestCase): + def setUp(self): + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.csrf_client = APIClient(enforce_csrf_checks=True) + self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) + self.vulnerability = Vulnerability.objects.create(summary="test") + self.affected_package, _ = Package.objects.get_or_create_from_purl( + purl="pkg:nginx/nginx@v3.4" + ) + self.fixed_package, _ = Package.objects.get_or_create_from_purl(purl="pkg:nginx/nginx@v4.0") + AffectedByPackageRelatedVulnerability.objects.create( + vulnerability=self.vulnerability, + created_by="test", + package=self.affected_package, + confidence=100, + ) + FixingPackageRelatedVulnerability.objects.create( + vulnerability=self.vulnerability, + created_by="test", + package=self.fixed_package, + confidence=100, + ) + for i in range(0, 10): + ref, _ = VulnerabilityReference.objects.get_or_create( + reference_id=f"cpe:/a:nginx:{i}", + url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:/a:nginx:{i}", + ) + VulnerabilityRelatedReference.objects.create( + reference=ref, vulnerability=self.vulnerability + ) + + def test_cpe_api(self): + response = self.csrf_client.get("/api/cpes/", format="json") + self.assertEqual(status.HTTP_200_OK, response.status_code) + + response_data = response.json() + self.assertEqual(1, response_data["count"]) + + class AliasApi(TestCase): def setUp(self): self.user = ApiUser.objects.create_api_user(username="e@mail.com") diff --git a/vulnerabilities/tests/test_api_extension.py b/vulnerabilities/tests/test_api_extension.py new file mode 100644 index 000000000..fc971a576 --- /dev/null +++ b/vulnerabilities/tests/test_api_extension.py @@ -0,0 +1,100 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path + +from pytest import fixture +from pytest import mark + +from vulnerabilities.api_extension import V2VulnerabilityReferenceSerializer +from vulnerabilities.api_extension import V2VulnerabilitySeveritySerializer +from vulnerabilities.models import Alias +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness +from vulnerabilities.tests.test_export import vulnerability_severity + +TEST_DATA_DIR = Path(__file__).parent / "test_data" / "apiv2" + +VCID = "VCID-pst6-b358-aaap" +PURL = "pkg:generic/nginx/test@2" + + +@fixture +def package(db): + return Package.objects.from_purl(PURL) + + +@fixture +def vulnerability_reference(): + return VulnerabilityReference.objects.create(reference_id="fake", url=f"https://..") + + +@fixture +def vulnerability_severity(vulnerability_reference): + return VulnerabilitySeverity.objects.create( + scoring_system="cvssv3_vector", + value="7.0", + scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + url=f"https://..", + ) + + +@fixture +def vulnerability(db, vulnerability_reference, vulnerability_severity): + vulnerability = Vulnerability.objects.create(vulnerability_id=VCID, summary="test-vuln") + Alias.objects.create(alias=f"CVE-xxx-xxx-xx", vulnerability=vulnerability) + + VulnerabilityRelatedReference.objects.create( + reference=vulnerability_reference, + vulnerability=vulnerability, + ) + + weakness = Weakness.objects.create(cwe_id=15) + vulnerability.weaknesses.add(weakness) + + return vulnerability + + +@fixture +def package_related_vulnerability(db, package, vulnerability): + AffectedByPackageRelatedVulnerability.objects.create( + package=package, + vulnerability=vulnerability, + ) + return package + + +@mark.django_db +def test_V2VulnerabilityReferenceSerializer(vulnerability_reference): + results = V2VulnerabilityReferenceSerializer(instance=vulnerability_reference).data + expected = {"reference_url": "https://..", "reference_id": "fake", "reference_type": ""} + assert expected == results + + +@mark.django_db +def test_V2VulnerabilitySeveritySerializer(vulnerability_severity): + results = V2VulnerabilitySeveritySerializer(instance=vulnerability_severity).data + expected = { + "published_at": None, + "url": "https://..", + "score": "7.0", + "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + "scoring_system": "cvssv3_vector", + } + + assert expected == results + + # purls_file = hashid.get_package_purls_yml_file_path(purl=PURL) + # results_pkgpurls = tmp_path / purls_file + # expected_pkgpurls = TEST_DATA_DIR / purls_file + # check_results_and_expected_files(results_pkgpurls, expected_pkgpurls) diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py new file mode 100644 index 000000000..fa3b7773c --- /dev/null +++ b/vulnerabilities/tests/test_api_v2.py @@ -0,0 +1,564 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.urls import reverse +from packageurl import PackageURL +from rest_framework import status +from rest_framework.test import APIClient +from rest_framework.test import APITestCase + +from vulnerabilities.api_v2 import PackageV2Serializer +from vulnerabilities.api_v2 import VulnerabilityListSerializer +from vulnerabilities.models import Alias +from vulnerabilities.models import ApiUser +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import Weakness + + +class VulnerabilityV2ViewSetTest(APITestCase): + def setUp(self): + # Create vulnerabilities + self.vuln1 = Vulnerability.objects.create( + vulnerability_id="VCID-1234", summary="Test vulnerability 1" + ) + self.vuln2 = Vulnerability.objects.create( + vulnerability_id="VCID-5678", summary="Test vulnerability 2" + ) + + # Create aliases + Alias.objects.create(alias="CVE-2021-1234", vulnerability=self.vuln1) + Alias.objects.create(alias="CVE-2021-5678", vulnerability=self.vuln2) + + # Create weaknesses + self.weakness1 = Weakness.objects.create(cwe_id=79) + self.weakness1.vulnerabilities.add(self.vuln1) + + self.weakness2 = Weakness.objects.create(cwe_id=89) + self.weakness2.vulnerabilities.add(self.vuln2) + + # Create references + self.reference1 = VulnerabilityReference.objects.create( + url="https://example.com/ref1", reference_type="advisory", reference_id="REF-1" + ) + self.reference1.vulnerabilities.add(self.vuln1) + + self.reference2 = VulnerabilityReference.objects.create( + url="https://example.com/ref2", reference_type="exploit", reference_id="REF-2" + ) + self.reference2.vulnerabilities.add(self.vuln2) + + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.client = APIClient(enforce_csrf_checks=True) + self.client.credentials(HTTP_AUTHORIZATION=self.auth) + + def test_list_vulnerabilities(self): + """ + Test listing vulnerabilities without filters. + Should return a paginated response with vulnerabilities dictionary. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + self.assertIn("VCID-1234", response.data["results"]["vulnerabilities"]) + self.assertIn("VCID-5678", response.data["results"]["vulnerabilities"]) + self.assertTrue("url" in response.data["results"]["vulnerabilities"]["VCID-1234"]) + + def test_retrieve_vulnerability_detail(self): + """ + Test retrieving vulnerability details by vulnerability_id. + """ + url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-1234"}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["vulnerability_id"], "VCID-1234") + self.assertEqual(response.data["summary"], "Test vulnerability 1") + self.assertEqual(response.data["aliases"], ["CVE-2021-1234"]) + self.assertEqual(len(response.data["weaknesses"]), 1) + self.assertEqual(len(response.data["references"]), 1) + + def test_filter_vulnerability_by_vulnerability_id(self): + """ + Test filtering vulnerabilities by vulnerability_id. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, {"vulnerability_id": "VCID-1234"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["vulnerability_id"], "VCID-1234") + + def test_filter_vulnerability_by_alias(self): + """ + Test filtering vulnerabilities by alias. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, {"alias": "CVE-2021-5678"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual( + response.data["results"]["vulnerabilities"]["VCID-5678"]["vulnerability_id"], + "VCID-5678", + ) + + def test_filter_vulnerabilities_multiple_ids(self): + """ + Test filtering vulnerabilities by multiple vulnerability_ids. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get( + url, {"vulnerability_id": ["VCID-1234", "VCID-5678"]}, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + + def test_filter_vulnerabilities_multiple_aliases(self): + """ + Test filtering vulnerabilities by multiple aliases. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get( + url, {"alias": ["CVE-2021-1234", "CVE-2021-5678"]}, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + + def test_invalid_vulnerability_id(self): + """ + Test retrieving a vulnerability with an invalid vulnerability_id. + Should return 404 Not Found. + """ + url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-9999"}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_get_url_in_serializer(self): + """ + Test that the serializer correctly includes the URL field. + """ + vulnerability = Vulnerability.objects.get(vulnerability_id="VCID-1234") + serializer = VulnerabilityListSerializer(vulnerability, context={"request": None}) + self.assertIn("url", serializer.data) + self.assertEqual(serializer.data["vulnerability_id"], "VCID-1234") + + def test_list_vulnerabilities_pagination(self): + """ + Test listing vulnerabilities with pagination. + """ + # Create additional vulnerabilities to trigger pagination + for i in range(3, 15): + Vulnerability.objects.create( + vulnerability_id=f"VCID-{i}", summary=f"Test vulnerability {i}" + ) + + url = reverse("vulnerability-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertIn("next", response.data) + self.assertIn("previous", response.data) + # The 'vulnerabilities' dictionary should contain vulnerabilities up to the page limit + self.assertEqual( + len(response.data["results"]["vulnerabilities"]), 10 + ) # Assuming default page size is 10 + + +class PackageV2ViewSetTest(APITestCase): + def setUp(self): + # Create packages + self.package1 = Package.objects.create( + package_url="pkg:pypi/django@3.2", name="django", version="3.2", type="pypi" + ) + self.package2 = Package.objects.create( + package_url="pkg:npm/lodash@4.17.20", name="lodash", version="4.17.20", type="npm" + ) + + # Create vulnerabilities + self.vuln1 = Vulnerability.objects.create( + vulnerability_id="VCID-1234", summary="Test vulnerability 1" + ) + self.vuln2 = Vulnerability.objects.create( + vulnerability_id="VCID-5678", summary="Test vulnerability 2" + ) + + # Associate packages with vulnerabilities + self.package1.affected_by_vulnerabilities.add(self.vuln1) + self.package2.fixing_vulnerabilities.add(self.vuln2) + + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.client = APIClient(enforce_csrf_checks=True) + self.client.credentials(HTTP_AUTHORIZATION=self.auth) + + def test_list_packages(self): + """ + Test listing packages without filters. + Should return a list of packages with their details and associated vulnerabilities. + """ + url = reverse("package-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("packages", response.data["results"]) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual(len(response.data["results"]["packages"]), 2) + # Verify that vulnerabilities are included + self.assertIsInstance(response.data["results"]["vulnerabilities"], dict) + package_vulns = set() + for package in response.data["results"]["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["results"]["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_filter_packages_by_purl(self): + """ + Test filtering packages by one or more PURLs. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"purl": "pkg:pypi/django@3.2"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") + + def test_filter_packages_by_affected_vulnerability(self): + """ + Test filtering packages by affected_by_vulnerability. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"affected_by_vulnerability": "VCID-1234"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") + + def test_filter_packages_by_fixing_vulnerability(self): + """ + Test filtering packages by fixing_vulnerability. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"fixing_vulnerability": "VCID-5678"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:npm/lodash@4.17.20") + + def test_package_serializer_fields(self): + """ + Test that the PackageV2Serializer returns the correct fields. + """ + package = Package.objects.get(package_url="pkg:pypi/django@3.2") + serializer = PackageV2Serializer(package) + data = serializer.data + self.assertIn("purl", data) + self.assertIn("affected_by_vulnerabilities", data) + self.assertIn("fixing_vulnerabilities", data) + self.assertIn("next_non_vulnerable_version", data) + self.assertIn("latest_non_vulnerable_version", data) + self.assertEqual(data["purl"], "pkg:pypi/django@3.2") + self.assertEqual(data["affected_by_vulnerabilities"], ["VCID-1234"]) + self.assertEqual(data["fixing_vulnerabilities"], []) + + def test_list_packages_pagination(self): + """ + Test listing packages with pagination. + """ + # Create additional packages to trigger pagination + for i in range(3, 15): + Package.objects.create( + package_url=f"pkg:pypi/package{i}@1.0.{i}", + name=f"package{i}", + version=f"1.0.{i}", + type="pypi", + ) + + url = reverse("package-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("packages", response.data["results"]) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertIn("next", response.data) + self.assertIn("previous", response.data) + self.assertEqual( + len(response.data["results"]["packages"]), 10 + ) # Assuming default page size is 10 + + def test_invalid_vulnerability_filter(self): + """ + Test filtering packages with an invalid vulnerability ID. + Should return an empty list. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"affected_by_vulnerability": "VCID-9999"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 0) + + def test_invalid_purl_filter(self): + """ + Test filtering packages with an invalid PURL. + Should return an empty list. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"purl": "pkg:nonexistent/package@1.0.0"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 0) + + def test_get_affected_by_vulnerabilities(self): + """ + Test the get_affected_by_vulnerabilities method in the serializer. + """ + package = Package.objects.get(package_url="pkg:pypi/django@3.2") + serializer = PackageV2Serializer() + vulnerabilities = serializer.get_affected_by_vulnerabilities(package) + self.assertEqual(vulnerabilities, ["VCID-1234"]) + + def test_get_fixing_vulnerabilities(self): + """ + Test the get_fixing_vulnerabilities method in the serializer. + """ + package = Package.objects.get(package_url="pkg:npm/lodash@4.17.20") + serializer = PackageV2Serializer() + vulnerabilities = serializer.get_fixing_vulnerabilities(package) + self.assertEqual(vulnerabilities, ["VCID-5678"]) + + def test_bulk_lookup_with_valid_purls(self): + """ + Test bulk lookup with valid PURLs. + Should return packages and their associated vulnerabilities. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + self.assertEqual(len(response.data["packages"]), 2) + # Verify that the returned data matches the packages + purls = [package["purl"] for package in response.data["packages"]] + self.assertIn("pkg:pypi/django@3.2", purls) + self.assertIn("pkg:npm/lodash@4.17.20", purls) + # Verify that vulnerabilities are included + package_vulns = set() + for package in response.data["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_bulk_lookup_with_invalid_purls(self): + """ + Test bulk lookup with invalid PURLs. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since the packages don't exist, the response should be empty + self.assertEqual(len(response.data["packages"]), 0) + self.assertEqual(len(response.data["vulnerabilities"]), 0) + + def test_bulk_lookup_with_empty_purls(self): + """ + Test bulk lookup with empty purls list. + Should return 400 Bad Request. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": []} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") + + def test_bulk_search_with_valid_purls(self): + """ + Test bulk search with valid PURLs. + Should return packages and their associated vulnerabilities. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + self.assertEqual(len(response.data["packages"]), 2) + purls = [package["purl"] for package in response.data["packages"]] + self.assertIn("pkg:pypi/django@3.2", purls) + self.assertIn("pkg:npm/lodash@4.17.20", purls) + # Verify that vulnerabilities are included + package_vulns = set() + for package in response.data["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_bulk_search_with_purl_only_true(self): + """ + Test bulk search with purl_only set to True. + Should return only the PURLs of vulnerable packages. + """ + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"], + "purl_only": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since purl_only=True, response should be a list of PURLs + self.assertIsInstance(response.data, list) + # Only vulnerable packages should be included + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) + + def test_bulk_search_with_plain_purl_true(self): + """ + Test bulk search with plain_purl set to True. + Should return packages grouped by plain PURLs. + """ + # Create another package with the same name and version but different qualifiers + Package.objects.create( + name="django", + version="3.2", + type="pypi", + qualifiers={"extension": "tar.gz"}, + ) + + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.2?extension=tar.gz"], + "plain_purl": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + # Since plain_purl=True, packages with the same type, namespace, name, version are grouped + self.assertEqual(len(response.data["packages"]), 1) + purl = response.data["packages"][0]["purl"] + self.assertTrue(purl.startswith("pkg:pypi/django@3.2")) + + def test_bulk_search_with_purl_only_and_plain_purl_true(self): + """ + Test bulk search with purl_only and plain_purl both set to True. + Should return only the plain PURLs of vulnerable packages. + """ + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.1"], + "purl_only": True, + "plain_purl": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Response should be a list of plain PURLs + self.assertIsInstance(response.data, list) + # Only one plain PURL should be returned for vulnerable packages + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) + + def test_bulk_search_with_invalid_purls(self): + """ + Test bulk search with invalid PURLs. + Should return an empty response. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since the packages don't exist, the response should be empty + self.assertEqual(len(response.data["packages"]), 0) + self.assertEqual(len(response.data["vulnerabilities"]), 0) + + def test_bulk_search_with_empty_purls(self): + """ + Test bulk search with empty purls list. + Should return 400 Bad Request. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": []} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") + + def test_all_vulnerable_packages(self): + """ + Test the 'all' endpoint that returns all vulnerable package URLs. + """ + url = reverse("package-v2-all") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since package1 is vulnerable, it should be returned + expected_purls = ["pkg:pypi/django@3.2"] + self.assertEqual(sorted(response.data), sorted(expected_purls)) + + def test_lookup_with_valid_purl(self): + """ + Test the 'lookup' endpoint with a valid PURL. + Should return the package and its associated vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "pkg:pypi/django@3.2"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(1, len(response.data)) + self.assertIn("purl", response.data[0]) + self.assertIn("affected_by_vulnerabilities", response.data[0]) + self.assertIn("fixing_vulnerabilities", response.data[0]) + self.assertIn("next_non_vulnerable_version", response.data[0]) + self.assertIn("latest_non_vulnerable_version", response.data[0]) + self.assertEqual(response.data[0]["purl"], "pkg:pypi/django@3.2") + self.assertEqual(response.data[0]["affected_by_vulnerabilities"], ["VCID-1234"]) + self.assertEqual(response.data[0]["fixing_vulnerabilities"], []) + + def test_lookup_with_invalid_purl(self): + """ + Test the 'lookup' endpoint with a PURL that does not exist. + Should return empty packages and vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "pkg:pypi/nonexistent@1.0.0"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # No packages or vulnerabilities should be returned + self.assertEqual(len(response.data), 0) + + def test_lookup_with_missing_purl(self): + """ + Test the 'lookup' endpoint without providing a 'purl'. + Should return 400 Bad Request. + """ + url = reverse("package-v2-lookup") + data = {} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A 'purl' is required.") + + def test_lookup_with_invalid_purl_format(self): + """ + Test the 'lookup' endpoint with an invalid PURL format. + Should return empty packages and vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "invalid_purl_format"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # No packages or vulnerabilities should be returned + self.assertEqual(len(response.data), 0) diff --git a/vulnerabilities/tests/test_archlinux.py b/vulnerabilities/tests/test_archlinux.py index f5ee38b75..d8582191c 100644 --- a/vulnerabilities/tests/test_archlinux.py +++ b/vulnerabilities/tests/test_archlinux.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_basics.py b/vulnerabilities/tests/test_basics.py index f56bf2173..c83aea393 100644 --- a/vulnerabilities/tests/test_basics.py +++ b/vulnerabilities/tests/test_basics.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_changelog.py b/vulnerabilities/tests/test_changelog.py index 5b3126a11..1d5eedaea 100644 --- a/vulnerabilities/tests/test_changelog.py +++ b/vulnerabilities/tests/test_changelog.py @@ -3,27 +3,28 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from datetime import datetime +from unittest.mock import patch import pytest +from packageurl import PackageURL from univers.version_range import NpmVersionRange from univers.versions import SemverVersion -from vulnerabilities.import_runner import ImportRunner +from vulnerabilities import models from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importers.npm import NpmImporter -from vulnerabilities.models import * +from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline @pytest.mark.django_db def test_package_changelog(): - pkg, _ = Package.objects.get_or_create_from_purl("pkg:npm/foo@1.0.0") - assert PackageChangeLog.objects.filter(package=pkg).count() == 0 - adv = Advisory.objects.create( - created_by=NpmImporter.qualified_name, + pkg, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@1.0.0") + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 0 + adv = models.Advisory.objects.create( + created_by=NpmImporterPipeline.pipeline_id, summary="TEST", date_collected=datetime.now(), url="https://test.com/source", @@ -38,18 +39,20 @@ def test_package_changelog(): ], aliases=["CVE-123"], ) - ImportRunner(NpmImporter).do_import([adv]) - assert PackageChangeLog.objects.filter(package=pkg).count() == 1 - ImportRunner(NpmImporter).do_import([adv]) - assert PackageChangeLog.objects.filter(package=pkg).count() == 1 + NpmImporterPipeline().import_advisory(advisory=adv) + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 + NpmImporterPipeline().import_advisory(advisory=adv) + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 assert ( - PackageChangeLog.objects.filter(action_type=PackageChangeLog.FIXING, package=pkg).count() + models.PackageChangeLog.objects.filter( + action_type=models.PackageChangeLog.FIXING, package=pkg + ).count() == 1 ) - pkg1, _ = Package.objects.get_or_create_from_purl("pkg:npm/foo@2.0.0") - assert PackageChangeLog.objects.filter(package=pkg1).count() == 0 - adv = Advisory.objects.create( - created_by=NpmImporter.qualified_name, + pkg1, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@2.0.0") + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 0 + adv = models.Advisory.objects.create( + created_by=NpmImporterPipeline.pipeline_id, summary="TEST-1", date_collected=datetime.now(), url="https://test.com/source-1", @@ -64,13 +67,14 @@ def test_package_changelog(): ], aliases=["CVE-145"], ) - ImportRunner(NpmImporter).do_import([adv]) - assert PackageChangeLog.objects.filter(package=pkg1).count() == 1 - ImportRunner(NpmImporter).do_import([adv]) - assert PackageChangeLog.objects.filter(package=pkg1).count() == 1 + NpmImporterPipeline().import_advisory(advisory=adv) + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 + NpmImporterPipeline().import_advisory(advisory=adv) + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 assert ( - PackageChangeLog.objects.filter( - action_type=PackageChangeLog.AFFECTED_BY, package=pkg1 + models.PackageChangeLog.objects.filter( + action_type=models.PackageChangeLog.AFFECTED_BY, + package=pkg1, ).count() == 1 ) @@ -78,8 +82,8 @@ def test_package_changelog(): @pytest.mark.django_db def test_vulnerability_changelog(): - adv = Advisory.objects.create( - created_by=NpmImporter.qualified_name, + adv = models.Advisory.objects.create( + created_by=NpmImporterPipeline.pipeline_id, summary="TEST_1", date_collected=datetime.now(), url="https://test.com/source", @@ -94,13 +98,40 @@ def test_vulnerability_changelog(): ], aliases=["CVE-TEST-1234"], ) - ImportRunner(NpmImporter).do_import([adv]) + NpmImporterPipeline().import_advisory(advisory=adv) # 1 Changelogs is expected here: # 1 for importing vuln details - assert VulnerabilityChangeLog.objects.count() == 1 - ImportRunner(NpmImporter).do_import([adv]) - assert VulnerabilityChangeLog.objects.count() == 1 + assert models.VulnerabilityChangeLog.objects.count() == 1 + NpmImporterPipeline().import_advisory(advisory=adv) + assert models.VulnerabilityChangeLog.objects.count() == 1 assert ( - VulnerabilityChangeLog.objects.filter(action_type=VulnerabilityChangeLog.IMPORT).count() + models.VulnerabilityChangeLog.objects.filter( + action_type=models.VulnerabilityChangeLog.IMPORT + ).count() == 1 ) + + +@patch("vulnerabilities.models.VULNERABLECODE_VERSION", "test-version") +@pytest.mark.django_db +def test_vulnerability_changelog_software_version(): + adv = models.Advisory.objects.create( + created_by=NpmImporterPipeline.pipeline_id, + summary="TEST_1", + date_collected=datetime.now(), + url="https://test.com/source", + affected_packages=[ + AffectedPackage( + package=PackageURL( + type="npm", + name="foo", + ), + fixed_version=SemverVersion("1.0"), + ).to_dict() + ], + aliases=["CVE-TEST-1234"], + ) + NpmImporterPipeline().import_advisory(advisory=adv) + npm_vulnerability_log = models.VulnerabilityChangeLog.objects.first() + + assert ("test-version", npm_vulnerability_log.software_version) diff --git a/vulnerabilities/tests/test_compute_package_version_rank.py b/vulnerabilities/tests/test_compute_package_version_rank.py new file mode 100644 index 000000000..12cd172a8 --- /dev/null +++ b/vulnerabilities/tests/test_compute_package_version_rank.py @@ -0,0 +1,59 @@ +from unittest.mock import patch + +import pytest +from univers.versions import Version + +from vulnerabilities.models import Package +from vulnerabilities.pipelines.compute_package_version_rank import ComputeVersionRankPipeline + + +@pytest.mark.django_db +class TestComputeVersionRankPipeline: + @pytest.fixture + def pipeline(self): + return ComputeVersionRankPipeline() + + @pytest.fixture + def packages(self, db): + package_type = "pypi" + namespace = "test_namespace" + name = "test_package" + Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.0.0") + Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.1.0") + Package.objects.create(type=package_type, namespace=namespace, name=name, version="0.9.0") + return Package.objects.filter(type=package_type, namespace=namespace, name=name) + + def test_compute_and_store_version_rank(self, pipeline, packages): + with patch.object(pipeline, "log") as mock_log: + pipeline.compute_and_store_version_rank() + assert mock_log.call_count > 0 + for package in packages: + assert package.version_rank is not None + + def test_update_version_rank_for_group(self, pipeline, packages): + with patch.object(Package.objects, "bulk_update") as mock_bulk_update: + pipeline.update_version_rank_for_group(packages) + mock_bulk_update.assert_called_once() + updated_packages = mock_bulk_update.call_args[0][0] + assert len(updated_packages) == len(packages) + for idx, package in enumerate(sorted(packages, key=lambda p: Version(p.version))): + assert updated_packages[idx].version_rank == idx + + def test_sort_packages_by_version(self, pipeline, packages): + sorted_packages = pipeline.sort_packages_by_version(packages) + versions = [p.version for p in sorted_packages] + assert versions == sorted(versions, key=Version) + + def test_sort_packages_by_version_empty(self, pipeline): + assert pipeline.sort_packages_by_version([]) == [] + + def test_sort_packages_by_version_invalid_scheme(self, pipeline, packages): + for package in packages: + package.type = "invalid" + assert pipeline.sort_packages_by_version(packages) == [] + + def test_compute_and_store_version_rank_invalid_scheme(self, pipeline): + Package.objects.create(type="invalid", namespace="test", name="package", version="1.0.0") + with patch.object(pipeline, "log") as mock_log: + pipeline.compute_and_store_version_rank() + mock_log.assert_any_call("Successfully populated `version_rank` for all packages.") diff --git a/vulnerabilities/tests/test_cpe_reference.py b/vulnerabilities/tests/test_cpe_reference.py index 7f119292f..2c66db2e8 100644 --- a/vulnerabilities/tests/test_cpe_reference.py +++ b/vulnerabilities/tests/test_cpe_reference.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_create_api_user_command.py b/vulnerabilities/tests/test_create_api_user_command.py index 6c54fca24..00faa9a78 100644 --- a/vulnerabilities/tests/test_create_api_user_command.py +++ b/vulnerabilities/tests/test_create_api_user_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_curl.py b/vulnerabilities/tests/test_curl.py new file mode 100644 index 000000000..6822e9677 --- /dev/null +++ b/vulnerabilities/tests/test_curl.py @@ -0,0 +1,73 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os +from unittest import TestCase +from unittest.mock import patch + +from vulnerabilities.importers.curl import get_cwe_from_curl_advisory +from vulnerabilities.importers.curl import parse_advisory_data +from vulnerabilities.tests import util_tests +from vulnerabilities.utils import load_json + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/curl") + + +class TestCurlImporter(TestCase): + def test_parse_advisory_data1(self): + mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock1.json")) + expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output1.json") + result = parse_advisory_data(mock_response) + result = result.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_parse_advisory_data2(self): + mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock2.json")) + expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output2.json") + result = parse_advisory_data(mock_response) + result = result.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_parse_advisory_data3(self): + mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock3.json")) + expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output3.json") + result = parse_advisory_data(mock_response) + result = result.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_get_cwe_from_curl_advisory(self): + assert get_cwe_from_curl_advisory( + { + "id": "CURL-CVE-2024-2466", + "database_specific": { + "CWE": { + "id": "CWE-297", + "desc": "Improper Validation of Certificate with Host Mismatch", + }, + }, + } + ) == [297] + + mock_advisory = [ + { + "id": "CURL-CVE-XXXX-XXXX", + "database_specific": {"CWE": {"id": "CWE-111111111", "desc": "Invalid weaknesses"}}, + }, + { + "id": "CURL-CVE-2024-2466", + "database_specific": { + "CWE": {"id": "CWE-311", "desc": "Missing Encryption of Sensitive Data"}, + }, + }, + ] + mock_cwe_list = [] + for advisory in mock_advisory: + mock_cwe_list.extend(get_cwe_from_curl_advisory(advisory)) + assert mock_cwe_list == [311] diff --git a/vulnerabilities/tests/test_data/curl/curl_advisory_mock1.json b/vulnerabilities/tests/test_data/curl/curl_advisory_mock1.json new file mode 100644 index 000000000..c84162ff6 --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/curl_advisory_mock1.json @@ -0,0 +1,61 @@ +{ + "schema_version": "1.5.0", + "id": "CURL-CVE-2024-2379", + "aliases": [ + "CVE-2024-2379" + ], + "summary": "QUIC certificate check bypass with wolfSSL", + "modified": "2024-03-26T10:36:00.00Z", + "database_specific": { + "package": "curl", + "URL": "https://curl.se/docs/CVE-2024-2379.json", + "www": "https://curl.se/docs/CVE-2024-2379.html", + "issue": "https://hackerone.com/reports/2410774", + "CWE": { + "id": "CWE-295", + "desc": "Improper Certificate Validation" + }, + "award": { + "amount": "540", + "currency": "USD" + }, + "last_affected": "8.6.0", + "severity": "Low" + }, + "published": "2024-03-27T08:00:00.00Z", + "affected": [ + { + "ranges": [ + { + "type": "SEMVER", + "events": [ + {"introduced": "8.6.0"}, + {"fixed": "8.7.0"} + ] + }, + { + "type": "GIT", + "repo": "https://github.com/curl/curl.git", + "events": [ + {"introduced": "5d044ad9480a9f556f4b6a252d7533b1ba7fe57e"}, + {"fixed": "aedbbdf18e689a5eee8dc39600914f5eda6c409c"} + ] + } + ], + "versions": [ + "8.6.0" + ] + } + ], + "credits": [ + { + "name": "Dexter Gerig", + "type": "FINDER" + }, + { + "name": "Daniel Stenberg", + "type": "REMEDIATION_DEVELOPER" + } + ], + "details": "libcurl skips the certificate verification for a QUIC connection under certain\nconditions, when built to use wolfSSL. If told to use an unknown/bad cipher or\ncurve, the error path accidentally skips the verification and returns OK, thus\nignoring any certificate problems." + } \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/curl/curl_advisory_mock2.json b/vulnerabilities/tests/test_data/curl/curl_advisory_mock2.json new file mode 100644 index 000000000..667ba758b --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/curl_advisory_mock2.json @@ -0,0 +1,61 @@ +{ + "schema_version": "1.5.0", + "id": "CURL-CVE-2024-0853", + "aliases": [ + "CVE-2024-0853" + ], + "summary": "OCSP verification bypass with TLS session reuse", + "modified": "2024-01-31T08:07:21.00Z", + "database_specific": { + "package": "curl", + "URL": "https://curl.se/docs/CVE-2024-0853.json", + "www": "https://curl.se/docs/CVE-2024-0853.html", + "issue": "https://hackerone.com/reports/2298922", + "CWE": { + "id": "CWE-299", + "desc": "Improper Check for Certificate Revocation" + }, + "award": { + "amount": "540", + "currency": "USD" + }, + "last_affected": "8.5.0", + "severity": "Low" + }, + "published": "2024-01-31T08:00:00.00Z", + "affected": [ + { + "ranges": [ + { + "type": "SEMVER", + "events": [ + {"introduced": "8.5.0"}, + {"fixed": "8.6.0"} + ] + }, + { + "type": "GIT", + "repo": "https://github.com/curl/curl.git", + "events": [ + {"introduced": "395365ad2d9a6c3f1a35d5e268a6af2824129832"}, + {"fixed": "c28e9478cb2548848eca9b765d0d409bfb18668c"} + ] + } + ], + "versions": [ + "8.5.0" + ] + } + ], + "credits": [ + { + "name": "Hiroki Kurosawa", + "type": "FINDER" + }, + { + "name": "Daniel Stenberg", + "type": "REMEDIATION_DEVELOPER" + } + ], + "details": "curl inadvertently kept the SSL session ID for connections in its cache even\nwhen the verify status (*OCSP stapling*) test failed. A subsequent transfer to\nthe same hostname could then succeed if the session ID cache was still fresh,\nwhich then skipped the verify status check." +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/curl/curl_advisory_mock3.json b/vulnerabilities/tests/test_data/curl/curl_advisory_mock3.json new file mode 100644 index 000000000..80b2c7388 --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/curl_advisory_mock3.json @@ -0,0 +1,71 @@ +{ + "schema_version": "1.5.0", + "id": "CURL-CVE-2023-46218", + "aliases": [ + "CVE-2023-46218" + ], + "summary": "cookie mixed case PSL bypass", + "modified": "2024-01-12T23:40:27.00Z", + "database_specific": { + "package": "curl", + "URL": "https://curl.se/docs/CVE-2023-46218.json", + "www": "https://curl.se/docs/CVE-2023-46218.html", + "issue": "https://hackerone.com/reports/2212193", + "CWE": { + "id": "CWE-201", + "desc": "Information Exposure Through Sent Data" + }, + "award": { + "amount": "2540", + "currency": "USD" + }, + "last_affected": "8.4.0", + "severity": "Medium" + }, + "published": "2023-12-06T08:00:00.00Z", + "affected": [ + { + "ranges": [ + { + "type": "SEMVER", + "events": [ + {"introduced": "7.46.0"}, + {"fixed": "8.5.0"} + ] + }, + { + "type": "GIT", + "repo": "https://github.com/curl/curl.git", + "events": [ + {"introduced": "e77b5b7453c1e8ccd7ec0816890d98e2f392e465"}, + {"fixed": "2b0994c29a721c91c572cff7808c572a24d251eb"} + ] + } + ], + "versions": [ + "8.4.0", "8.3.0", "8.2.1", "8.2.0", "8.1.2", "8.1.1", "8.1.0", + "8.0.1", "8.0.0", "7.88.1", "7.88.0", "7.87.0", "7.86.0", "7.85.0", + "7.84.0", "7.83.1", "7.83.0", "7.82.0", "7.81.0", "7.80.0", "7.79.1", + "7.79.0", "7.78.0", "7.77.0", "7.76.1", "7.76.0", "7.75.0", "7.74.0", + "7.73.0", "7.72.0", "7.71.1", "7.71.0", "7.70.0", "7.69.1", "7.69.0", + "7.68.0", "7.67.0", "7.66.0", "7.65.3", "7.65.2", "7.65.1", "7.65.0", + "7.64.1", "7.64.0", "7.63.0", "7.62.0", "7.61.1", "7.61.0", "7.60.0", + "7.59.0", "7.58.0", "7.57.0", "7.56.1", "7.56.0", "7.55.1", "7.55.0", + "7.54.1", "7.54.0", "7.53.1", "7.53.0", "7.52.1", "7.52.0", "7.51.0", + "7.50.3", "7.50.2", "7.50.1", "7.50.0", "7.49.1", "7.49.0", "7.48.0", + "7.47.1", "7.47.0", "7.46.0" + ] + } + ], + "credits": [ + { + "name": "Harry Sintonen", + "type": "FINDER" + }, + { + "name": "Daniel Stenberg", + "type": "REMEDIATION_DEVELOPER" + } + ], + "details": "This flaw allows a malicious HTTP server to set \"super cookies\" in curl that\nare then passed back to more origins than what is otherwise allowed or\npossible. This allows a site to set cookies that then would get sent to\ndifferent and unrelated sites and domains.\n\nIt could do this by exploiting a mixed case flaw in curl's function that\nverifies a given cookie domain against the Public Suffix List (PSL). For\nexample a cookie could be set with `domain=co.UK` when the URL used a\nlowercase hostname `curl.co.uk`, even though `co.uk` is listed as a PSL\ndomain." + } \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output1.json b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output1.json new file mode 100644 index 000000000..f0bfd19a2 --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output1.json @@ -0,0 +1,45 @@ +{ + "aliases": [ + "CVE-2024-2379" + ], + "summary": "QUIC certificate check bypass with wolfSSL", + "affected_packages": [ + { + "package": { + "type": "generic", + "namespace": "curl.se", + "name": "curl", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:generic/8.6.0", + "fixed_version": "8.7.0" + } + ], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://curl.se/docs/CVE-2024-2379.html", + "severities": [ + { + "system": "cvssv3.1", + "value": "Low", + "scoring_elements": "" + } + ] + }, + { + "reference_id": "", + "reference_type": "", + "url": "https://hackerone.com/reports/2410774", + "severities": [] + } + ], + "date_published": "2024-03-27T08:00:00+00:00", + "weaknesses": [ + 295 + ], + "url": "https://curl.se/docs/CVE-2024-2379.json" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output2.json b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output2.json new file mode 100644 index 000000000..797dcea6c --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output2.json @@ -0,0 +1,45 @@ +{ + "aliases": [ + "CVE-2024-0853" + ], + "summary": "OCSP verification bypass with TLS session reuse", + "affected_packages": [ + { + "package": { + "type": "generic", + "namespace": "curl.se", + "name": "curl", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:generic/8.5.0", + "fixed_version": "8.6.0" + } + ], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://curl.se/docs/CVE-2024-0853.html", + "severities": [ + { + "system": "cvssv3.1", + "value": "Low", + "scoring_elements": "" + } + ] + }, + { + "reference_id": "", + "reference_type": "", + "url": "https://hackerone.com/reports/2298922", + "severities": [] + } + ], + "date_published": "2024-01-31T08:00:00+00:00", + "weaknesses": [ + 299 + ], + "url": "https://curl.se/docs/CVE-2024-0853.json" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output3.json b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output3.json new file mode 100644 index 000000000..ff31e6c36 --- /dev/null +++ b/vulnerabilities/tests/test_data/curl/expected_curl_advisory_output3.json @@ -0,0 +1,45 @@ +{ + "aliases": [ + "CVE-2023-46218" + ], + "summary": "cookie mixed case PSL bypass", + "affected_packages": [ + { + "package": { + "type": "generic", + "namespace": "curl.se", + "name": "curl", + "version": "", + "qualifiers": "", + "subpath": "" + }, + "affected_version_range": "vers:generic/7.46.0|7.47.0|7.47.1|7.48.0|7.49.0|7.49.1|7.50.0|7.50.1|7.50.2|7.50.3|7.51.0|7.52.0|7.52.1|7.53.0|7.53.1|7.54.0|7.54.1|7.55.0|7.55.1|7.56.0|7.56.1|7.57.0|7.58.0|7.59.0|7.60.0|7.61.0|7.61.1|7.62.0|7.63.0|7.64.0|7.64.1|7.65.0|7.65.1|7.65.2|7.65.3|7.66.0|7.67.0|7.68.0|7.69.0|7.69.1|7.70.0|7.71.0|7.71.1|7.72.0|7.73.0|7.74.0|7.75.0|7.76.0|7.76.1|7.77.0|7.78.0|7.79.0|7.79.1|7.80.0|7.81.0|7.82.0|7.83.0|7.83.1|7.84.0|7.85.0|7.86.0|7.87.0|7.88.0|7.88.1|8.0.0|8.0.1|8.1.0|8.1.1|8.1.2|8.2.0|8.2.1|8.3.0|8.4.0", + "fixed_version": "8.5.0" + } + ], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://curl.se/docs/CVE-2023-46218.html", + "severities": [ + { + "system": "cvssv3.1", + "value": "Medium", + "scoring_elements": "" + } + ] + }, + { + "reference_id": "", + "reference_type": "", + "url": "https://hackerone.com/reports/2212193", + "severities": [] + } + ], + "date_published": "2023-12-06T08:00:00+00:00", + "weaknesses": [ + 201 + ], + "url": "https://curl.se/docs/CVE-2023-46218.json" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv new file mode 100644 index 000000000..a63701d8c --- /dev/null +++ b/vulnerabilities/tests/test_data/exploitdb_improver/files_exploits.csv @@ -0,0 +1,2 @@ +id,file,description,date_published,author,type,platform,port,date_added,date_updated,verified,codes,tags,aliases,screenshot_url,application_url,source_url +16929,exploits/aix/dos/16929.rb,"AIX Calendar Manager Service Daemon (rpc.cmsd) Opcode 21 - Buffer Overflow (Metasploit)",2010-11-11,Metasploit,dos,aix,,2010-11-11,2011-03-06,1,CVE-2009-3699;OSVDB-58726,"Metasploit Framework (MSF)",,,,http://aix.software.ibm.com/aix/efixes/security/cmsd_advisory.asc diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/purls.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/purls.yml new file mode 100644 index 000000000..a2a1c66e8 --- /dev/null +++ b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/purls.yml @@ -0,0 +1 @@ +- pkg:generic/nginx/test@2 diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/vulnerabilities.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/vulnerabilities.yml new file mode 100644 index 000000000..c4c0690d1 --- /dev/null +++ b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/vulnerabilities.yml @@ -0,0 +1,4 @@ +- purl: pkg:generic/nginx/test@2 + affected_by_vulnerabilities: + - VCID-pst6-b358-aaap + fixing_vulnerabilities: [] diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml new file mode 100644 index 000000000..63ab7f5af --- /dev/null +++ b/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml @@ -0,0 +1,16 @@ +vulnerability_id: VCID-pst6-b358-aaap +aliases: + - CVE-xxx-xxx-xx +summary: test-vuln +severities: + - score: '7.0' + scoring_system: cvssv3_vector + scoring_elements: CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H + published_at: + url: https://.. +weaknesses: + - CWE-15 +references: + - url: https://.. + reference_type: + reference_id: fake diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2002-0001-expected.json b/vulnerabilities/tests/test_data/gsd/GSD-2002-0001-expected.json new file mode 100644 index 000000000..60f1c9f31 --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2002-0001-expected.json @@ -0,0 +1,105 @@ +{ + "aliases": [ + "CVE-2002-0001", + "GSD-2002-0001" + ], + "summary": "Vulnerability in RFC822 address parser in mutt before 1.2.5.1 and mutt 1.3.x before 1.3.25 allows remote attackers to execute arbitrary commands via an improperly terminated comment or phrase in the address list.", + "affected_packages": [ + + ], + "references": [ + { + "reference_id": "", + "reference_type" : "", + "url": "http://online.securityfocus.com/advisories/3778", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "ftp://ftp.freebsd.org/pub/FreeBSD/CERT/advisories/FreeBSD-SA-02:04.mutt.asc", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.debian.org/security/2002/dsa-096", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.mutt.org/announce/mutt-1.2.5.1-1.3.25.html", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://distro.conectiva.com.br/atualizacoes/?id=a&anuncio=000449", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.iss.net/security_center/static/7759.php", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.securityfocus.com/bid/3774", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "ftp://ftp.caldera.com/pub/security/OpenLinux/CSSA-2002-002.0.txt", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.novell.com/linux/security/advisories/2002_001_mutt_txt.html", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://marc.info/?l=bugtraq&m=100994648918287&w=2", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "http://www.redhat.com/support/errata/RHSA-2002-003.html", + "severities": [ + + ] + } + ], + "date_published": "2002-02-27T05:00:00+00:00", + "weaknesses": [ + + ], + "url": "" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2002-0001.json b/vulnerabilities/tests/test_data/gsd/GSD-2002-0001.json new file mode 100644 index 000000000..15fd3c33f --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2002-0001.json @@ -0,0 +1,277 @@ +{ + "GSD": { + "alias": "CVE-2002-0001", + "description": "Vulnerability in RFC822 address parser in mutt before 1.2.5.1 and mutt 1.3.x before 1.3.25 allows remote attackers to execute arbitrary commands via an improperly terminated comment or phrase in the address list.", + "id": "GSD-2002-0001", + "references": [ + "https://www.debian.org/security/2002/dsa-096", + "https://access.redhat.com/errata/RHSA-2002:003" + ] + }, + "namespaces": { + "cve.org": { + "CVE_data_meta": { + "ASSIGNER": "cve@mitre.org", + "ID": "CVE-2002-0001", + "STATE": "PUBLIC" + }, + "affects": { + "vendor": { + "vendor_data": [ + { + "product": { + "product_data": [ + { + "product_name": "n/a", + "version": { + "version_data": [ + { + "version_value": "n/a" + } + ] + } + } + ] + }, + "vendor_name": "n/a" + } + ] + } + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "eng", + "value": "Vulnerability in RFC822 address parser in mutt before 1.2.5.1 and mutt 1.3.x before 1.3.25 allows remote attackers to execute arbitrary commands via an improperly terminated comment or phrase in the address list." + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "eng", + "value": "n/a" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "HPSBTL0201-011", + "refsource": "HP", + "url": "http://online.securityfocus.com/advisories/3778" + }, + { + "name": "FreeBSD-SA-02:04", + "refsource": "FREEBSD", + "url": "ftp://ftp.freebsd.org/pub/FreeBSD/CERT/advisories/FreeBSD-SA-02:04.mutt.asc" + }, + { + "name": "DSA-096", + "refsource": "DEBIAN", + "url": "http://www.debian.org/security/2002/dsa-096" + }, + { + "name": "http://www.mutt.org/announce/mutt-1.2.5.1-1.3.25.html", + "refsource": "CONFIRM", + "url": "http://www.mutt.org/announce/mutt-1.2.5.1-1.3.25.html" + }, + { + "name": "CLA-2002:449", + "refsource": "CONECTIVA", + "url": "http://distro.conectiva.com.br/atualizacoes/?id=a&anuncio=000449" + }, + { + "name": "mutt-address-handling-bo(7759)", + "refsource": "XF", + "url": "http://www.iss.net/security_center/static/7759.php" + }, + { + "name": "3774", + "refsource": "BID", + "url": "http://www.securityfocus.com/bid/3774" + }, + { + "name": "CSSA-2002-002.0", + "refsource": "CALDERA", + "url": "ftp://ftp.caldera.com/pub/security/OpenLinux/CSSA-2002-002.0.txt" + }, + { + "name": "SuSE-SA:2002:001", + "refsource": "SUSE", + "url": "http://www.novell.com/linux/security/advisories/2002_001_mutt_txt.html" + }, + { + "name": "20020101 [Announce] SECURITY: mutt-1.2.5.1 and mutt-1.3.25 released.", + "refsource": "BUGTRAQ", + "url": "http://marc.info/?l=bugtraq&m=100994648918287&w=2" + }, + { + "name": "RHSA-2002:003", + "refsource": "REDHAT", + "url": "http://www.redhat.com/support/errata/RHSA-2002-003.html" + } + ] + } + }, + "nvd.nist.gov": { + "configurations": { + "CVE_data_version": "4.0", + "nodes": [ + { + "children": [], + "cpe_match": [ + { + "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", + "cpe_name": [], + "versionEndIncluding": "1.2.5.1", + "vulnerable": true + }, + { + "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", + "cpe_name": [], + "versionEndIncluding": "1.3.25", + "vulnerable": true + } + ], + "operator": "OR" + } + ] + }, + "cve": { + "CVE_data_meta": { + "ASSIGNER": "cve@mitre.org", + "ID": "CVE-2002-0001" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "en", + "value": "Vulnerability in RFC822 address parser in mutt before 1.2.5.1 and mutt 1.3.x before 1.3.25 allows remote attackers to execute arbitrary commands via an improperly terminated comment or phrase in the address list." + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "en", + "value": "NVD-CWE-Other" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "DSA-096", + "refsource": "DEBIAN", + "tags": [ + "Patch" + ], + "url": "http://www.debian.org/security/2002/dsa-096" + }, + { + "name": "RHSA-2002:003", + "refsource": "REDHAT", + "tags": [ + "Patch" + ], + "url": "http://www.redhat.com/support/errata/RHSA-2002-003.html" + }, + { + "name": "http://www.mutt.org/announce/mutt-1.2.5.1-1.3.25.html", + "refsource": "CONFIRM", + "tags": [], + "url": "http://www.mutt.org/announce/mutt-1.2.5.1-1.3.25.html" + }, + { + "name": "SuSE-SA:2002:001", + "refsource": "SUSE", + "tags": [], + "url": "http://www.novell.com/linux/security/advisories/2002_001_mutt_txt.html" + }, + { + "name": "FreeBSD-SA-02:04", + "refsource": "FREEBSD", + "tags": [], + "url": "ftp://ftp.freebsd.org/pub/FreeBSD/CERT/advisories/FreeBSD-SA-02:04.mutt.asc" + }, + { + "name": "HPSBTL0201-011", + "refsource": "HP", + "tags": [], + "url": "http://online.securityfocus.com/advisories/3778" + }, + { + "name": "CSSA-2002-002.0", + "refsource": "CALDERA", + "tags": [], + "url": "ftp://ftp.caldera.com/pub/security/OpenLinux/CSSA-2002-002.0.txt" + }, + { + "name": "3774", + "refsource": "BID", + "tags": [], + "url": "http://www.securityfocus.com/bid/3774" + }, + { + "name": "mutt-address-handling-bo(7759)", + "refsource": "XF", + "tags": [], + "url": "http://www.iss.net/security_center/static/7759.php" + }, + { + "name": "CLA-2002:449", + "refsource": "CONECTIVA", + "tags": [], + "url": "http://distro.conectiva.com.br/atualizacoes/?id=a&anuncio=000449" + }, + { + "name": "20020101 [Announce] SECURITY: mutt-1.2.5.1 and mutt-1.3.25 released.", + "refsource": "BUGTRAQ", + "tags": [], + "url": "http://marc.info/?l=bugtraq&m=100994648918287&w=2" + } + ] + } + }, + "impact": { + "baseMetricV2": { + "cvssV2": { + "accessComplexity": "LOW", + "accessVector": "NETWORK", + "authentication": "NONE", + "availabilityImpact": "PARTIAL", + "baseScore": 7.5, + "confidentialityImpact": "PARTIAL", + "integrityImpact": "PARTIAL", + "vectorString": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "version": "2.0" + }, + "exploitabilityScore": 10.0, + "impactScore": 6.4, + "obtainAllPrivilege": false, + "obtainOtherPrivilege": false, + "obtainUserPrivilege": false, + "severity": "HIGH", + "userInteractionRequired": false + } + }, + "lastModifiedDate": "2016-10-18T02:15Z", + "publishedDate": "2002-02-27T05:00Z" + } + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2006-0326-expected.json b/vulnerabilities/tests/test_data/gsd/GSD-2006-0326-expected.json new file mode 100644 index 000000000..c9c1308c6 --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2006-0326-expected.json @@ -0,0 +1,18 @@ +{ + "aliases": [ + "CVE-2006-0326", + "GSD-2006-0326" + ], + "summary": "** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem. When the candidate has been publicized, the details for this candidate will be provided.", + "affected_packages": [ + + ], + "references": [ + + ], + "date_published": null, + "weaknesses": [ + + ], + "url": "" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2006-0326.json b/vulnerabilities/tests/test_data/gsd/GSD-2006-0326.json new file mode 100644 index 000000000..cbe477995 --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2006-0326.json @@ -0,0 +1,26 @@ +{ + "GSD": { + "alias": "CVE-2006-0326", + "id": "GSD-2006-0326" + }, + "namespaces": { + "cve.org": { + "CVE_data_meta": { + "ASSIGNER": "cve@mitre.org", + "ID": "CVE-2006-0326", + "STATE": "RESERVED" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "eng", + "value": "** RESERVED ** This candidate has been reserved by an organization or individual that will use it when announcing a new security problem. When the candidate has been publicized, the details for this candidate will be provided." + } + ] + } + } + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2016-20005-expected.json b/vulnerabilities/tests/test_data/gsd/GSD-2016-20005-expected.json new file mode 100644 index 000000000..d7f07bcda --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2016-20005-expected.json @@ -0,0 +1,16 @@ +{ + "aliases": ["CVE-2016-20005", "GSD-2016-20005"], + "summary": "The REST/JSON project 7.x-1.x for Drupal allows user registration bypass, aka SA-CONTRIB-2016-033. NOTE: This project is not covered by Drupal's security advisory policy.", + "affected_packages": [], + "references": [ + { + "reference_id": "", + "reference_type" : "", + "url": "https://www.drupal.org/node/2744889", + "severities": [] + } + ], + "date_published": "2021-01-01T01:15:00+00:00", + "weaknesses": [], + "url": "" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2016-20005.json b/vulnerabilities/tests/test_data/gsd/GSD-2016-20005.json new file mode 100644 index 000000000..ac54ce17c --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2016-20005.json @@ -0,0 +1,174 @@ +{ + "GSD": { + "alias": "CVE-2016-20005", + "description": "The REST/JSON project 7.x-1.x for Drupal allows user registration bypass, aka SA-CONTRIB-2016-033. NOTE: This project is not covered by Drupal's security advisory policy.", + "id": "GSD-2016-20005" + }, + "namespaces": { + "cve.org": { + "CVE_data_meta": { + "ASSIGNER": "cve@mitre.org", + "ID": "CVE-2016-20005", + "STATE": "PUBLIC" + }, + "affects": { + "vendor": { + "vendor_data": [ + { + "product": { + "product_data": [ + { + "product_name": "n/a", + "version": { + "version_data": [ + { + "version_value": "n/a" + } + ] + } + } + ] + }, + "vendor_name": "n/a" + } + ] + } + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "eng", + "value": "The REST/JSON project 7.x-1.x for Drupal allows user registration bypass, aka SA-CONTRIB-2016-033. NOTE: This project is not covered by Drupal's security advisory policy." + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "eng", + "value": "n/a" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "https://www.drupal.org/node/2744889", + "refsource": "MISC", + "url": "https://www.drupal.org/node/2744889" + } + ] + } + }, + "nvd.nist.gov": { + "configurations": { + "CVE_data_version": "4.0", + "nodes": [ + { + "children": [], + "cpe_match": [ + { + "cpe23Uri": "cpe:2.3:a:rest\\/json_project:rest\\/json:*:*:*:*:*:drupal:*:*", + "cpe_name": [], + "versionEndIncluding": "7.x-1.5", + "vulnerable": true + } + ], + "operator": "OR" + } + ] + }, + "cve": { + "CVE_data_meta": { + "ASSIGNER": "cve@mitre.org", + "ID": "CVE-2016-20005" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "en", + "value": "The REST/JSON project 7.x-1.x for Drupal allows user registration bypass, aka SA-CONTRIB-2016-033. NOTE: This project is not covered by Drupal's security advisory policy." + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "en", + "value": "CWE-863" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "https://www.drupal.org/node/2744889", + "refsource": "MISC", + "tags": [ + "Third Party Advisory" + ], + "url": "https://www.drupal.org/node/2744889" + } + ] + } + }, + "impact": { + "baseMetricV2": { + "acInsufInfo": false, + "cvssV2": { + "accessComplexity": "LOW", + "accessVector": "NETWORK", + "authentication": "NONE", + "availabilityImpact": "PARTIAL", + "baseScore": 7.5, + "confidentialityImpact": "PARTIAL", + "integrityImpact": "PARTIAL", + "vectorString": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "version": "2.0" + }, + "exploitabilityScore": 10.0, + "impactScore": 6.4, + "obtainAllPrivilege": false, + "obtainOtherPrivilege": false, + "obtainUserPrivilege": false, + "severity": "HIGH", + "userInteractionRequired": false + }, + "baseMetricV3": { + "cvssV3": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "confidentialityImpact": "HIGH", + "integrityImpact": "HIGH", + "privilegesRequired": "NONE", + "scope": "UNCHANGED", + "userInteraction": "NONE", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1" + }, + "exploitabilityScore": 3.9, + "impactScore": 5.9 + } + }, + "lastModifiedDate": "2021-01-07T14:59Z", + "publishedDate": "2021-01-01T01:15Z" + } + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2022-4030-expected.json b/vulnerabilities/tests/test_data/gsd/GSD-2022-4030-expected.json new file mode 100644 index 000000000..708c84ffa --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2022-4030-expected.json @@ -0,0 +1,33 @@ +{ + "aliases": [ + "CVE-2022-4030", + "GSD-2022-4030" + ], + "summary": "The Simple:Press plugin for WordPress is vulnerable to Path Traversal in versions up to, and including, 6.8 via the 'file' parameter which can be manipulated during user avatar deletion. This makes it possible with attackers, with minimal permissions such as a subscriber, to supply paths to arbitrary files on the server that will subsequently be deleted. This can be used to delete the wp-config.php file that can allow an attacker to configure the site and achieve remote code execution.", + "affected_packages": [ + + ], + "references": [ + { + "reference_id": "", + "reference_type" : "", + "url": "https://plugins.trac.wordpress.org/changeset?sfp_email=&sfph_mail=&reponame=&old=2804020%40simplepress&new=2804020%40simplepress&sfp_email=&sfph_mail=", + "severities": [ + + ] + }, + { + "reference_id": "", + "reference_type" : "", + "url": "https://www.wordfence.com/vulnerability-advisories-continued/#CVE-2022-4030", + "severities": [ + + ] + } + ], + "date_published": "2022-11-29T21:15:00+00:00", + "weaknesses": [ + + ], + "url": "" +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2022-4030.json b/vulnerabilities/tests/test_data/gsd/GSD-2022-4030.json new file mode 100644 index 000000000..a8f44c841 --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2022-4030.json @@ -0,0 +1,188 @@ +{ + "GSD": { + "alias": "CVE-2022-4030", + "description": "The Simple:Press plugin for WordPress is vulnerable to Path Traversal in versions up to, and including, 6.8 via the 'file' parameter which can be manipulated during user avatar deletion. This makes it possible with attackers, with minimal permissions such as a subscriber, to supply paths to arbitrary files on the server that will subsequently be deleted. This can be used to delete the wp-config.php file that can allow an attacker to configure the site and achieve remote code execution.", + "id": "GSD-2022-4030" + }, + "namespaces": { + "cve.org": { + "CVE_data_meta": { + "ASSIGNER": "security@wordfence.com", + "ID": "CVE-2022-4030", + "STATE": "PUBLIC" + }, + "affects": { + "vendor": { + "vendor_data": [ + { + "product": { + "product_data": [ + { + "product_name": "Simple:Press \u2013 WordPress Forum Plugin", + "version": { + "version_data": [ + { + "version_affected": "=", + "version_value": "*" + } + ] + } + } + ] + }, + "vendor_name": "simplepress" + } + ] + } + }, + "credits": [ + { + "lang": "en", + "value": "Luca Greeb" + }, + { + "lang": "en", + "value": "Andreas Kr\u00fcger" + } + ], + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "eng", + "value": "The Simple:Press plugin for WordPress is vulnerable to Path Traversal in versions up to, and including, 6.8 via the 'file' parameter which can be manipulated during user avatar deletion. This makes it possible with attackers, with minimal permissions such as a subscriber, to supply paths to arbitrary files on the server that will subsequently be deleted. This can be used to delete the wp-config.php file that can allow an attacker to configure the site and achieve remote code execution." + } + ] + }, + "impact": { + "cvss": [ + { + "baseScore": 8.1, + "baseSeverity": "HIGH", + "vectorString": "CVSS:3.1/A:H/I:H/C:N/S:U/UI:N/PR:L/AC:L/AV:N", + "version": "3.1" + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "eng", + "value": "CWE-22 Improper Limitation of a Pathname to a Restricted Directory ('Path Traversal')" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "https://plugins.trac.wordpress.org/changeset?sfp_email=&sfph_mail=&reponame=&old=2804020%40simplepress&new=2804020%40simplepress&sfp_email=&sfph_mail=", + "refsource": "MISC", + "url": "https://plugins.trac.wordpress.org/changeset?sfp_email=&sfph_mail=&reponame=&old=2804020%40simplepress&new=2804020%40simplepress&sfp_email=&sfph_mail=" + }, + { + "name": "https://www.wordfence.com/vulnerability-advisories-continued/#CVE-2022-4030", + "refsource": "MISC", + "url": "https://www.wordfence.com/vulnerability-advisories-continued/#CVE-2022-4030" + } + ] + } + }, + "nvd.nist.gov": { + "configurations": { + "CVE_data_version": "4.0", + "nodes": [ + { + "children": [], + "cpe_match": [ + { + "cpe23Uri": "cpe:2.3:a:simple-press:simple\\:press:*:*:*:*:*:wordpress:*:*", + "cpe_name": [], + "versionEndIncluding": "6.8.0", + "vulnerable": true + } + ], + "operator": "OR" + } + ] + }, + "cve": { + "CVE_data_meta": { + "ASSIGNER": "security@wordfence.com", + "ID": "CVE-2022-4030" + }, + "data_format": "MITRE", + "data_type": "CVE", + "data_version": "4.0", + "description": { + "description_data": [ + { + "lang": "en", + "value": "The Simple:Press plugin for WordPress is vulnerable to Path Traversal in versions up to, and including, 6.8 via the 'file' parameter which can be manipulated during user avatar deletion. This makes it possible with attackers, with minimal permissions such as a subscriber, to supply paths to arbitrary files on the server that will subsequently be deleted. This can be used to delete the wp-config.php file that can allow an attacker to configure the site and achieve remote code execution." + } + ] + }, + "problemtype": { + "problemtype_data": [ + { + "description": [ + { + "lang": "en", + "value": "CWE-22" + } + ] + } + ] + }, + "references": { + "reference_data": [ + { + "name": "https://plugins.trac.wordpress.org/changeset?sfp_email=&sfph_mail=&reponame=&old=2804020%40simplepress&new=2804020%40simplepress&sfp_email=&sfph_mail=", + "refsource": "MISC", + "tags": [ + "Patch", + "Third Party Advisory" + ], + "url": "https://plugins.trac.wordpress.org/changeset?sfp_email=&sfph_mail=&reponame=&old=2804020%40simplepress&new=2804020%40simplepress&sfp_email=&sfph_mail=" + }, + { + "name": "https://www.wordfence.com/vulnerability-advisories-continued/#CVE-2022-4030", + "refsource": "MISC", + "tags": [ + "Third Party Advisory" + ], + "url": "https://www.wordfence.com/vulnerability-advisories-continued/#CVE-2022-4030" + } + ] + } + }, + "impact": { + "baseMetricV3": { + "cvssV3": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 8.1, + "baseSeverity": "HIGH", + "confidentialityImpact": "NONE", + "integrityImpact": "HIGH", + "privilegesRequired": "LOW", + "scope": "UNCHANGED", + "userInteraction": "NONE", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:H", + "version": "3.1" + }, + "exploitabilityScore": 2.8, + "impactScore": 5.2 + } + }, + "lastModifiedDate": "2022-12-01T18:41Z", + "publishedDate": "2022-11-29T21:15Z" + } + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/gsd/GSD-2023-1000387.json b/vulnerabilities/tests/test_data/gsd/GSD-2023-1000387.json new file mode 100644 index 000000000..44b7ec42a --- /dev/null +++ b/vulnerabilities/tests/test_data/gsd/GSD-2023-1000387.json @@ -0,0 +1,61 @@ +{ + "GSD": { + "vendor_name": "Linux", + "product_name": "Kernel", + "product_version": "versions from v6.0 to before v6.0.19", + "vulnerability_type": "unspecified", + "affected_component": "unspecified", + "attack_vector": "unspecified", + "impact": "unspecified", + "credit": "", + "references": [ + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=0f28cca87e9afc22280c44d378d2a6e249933977", + "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/?id=2d5a6742a242091292cc0a2b607be701a45d0c4e" + ], + "extended_references": [ + { + "type": "commit", + "value": "0f28cca87e9afc22280c44d378d2a6e249933977", + "note": "introduced" + }, + { + "type": "commit", + "value": "2d5a6742a242091292cc0a2b607be701a45d0c4e", + "note": "fixed" + } + ], + "reporter": "joshbressers", + "reporter_id": 1692786, + "notes": "", + "description": "drm/amdkfd: Fix kernel warning during topology setup\n\nThis is an automated ID intended to aid in discovery of potential security vulnerabilities. The actual impact and attack plausibility have not yet been proven.\nThis ID is fixed in Linux Kernel version v6.0.19 by commit 2d5a6742a242091292cc0a2b607be701a45d0c4e, it was introduced in version v6.0 by commit 0f28cca87e9afc22280c44d378d2a6e249933977. For more details please see the references link." + }, + "OSV": { + "id": "GSD-2023-1000387", + "modified": "2023-01-17T17:35:43.401817Z", + "published": "2023-01-17T17:35:43.401817Z", + "summary": "drm/amdkfd: Fix kernel warning during topology setup", + "details": "drm/amdkfd: Fix kernel warning during topology setup\n\nThis is an automated ID intended to aid in discovery of potential security vulnerabilities. The actual impact and attack plausibility have not yet been proven.\nThis ID is fixed in Linux Kernel version v6.0.19 by commit 2d5a6742a242091292cc0a2b607be701a45d0c4e, it was introduced in version v6.0 by commit 0f28cca87e9afc22280c44d378d2a6e249933977. For more details please see the references link.", + "affected": [ + { + "package": { + "name": "Kernel", + "ecosystem": "Linux" + }, + "ranges": [ + { + "type": "GIT", + "repo": "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/", + "events": [ + { + "introduced": "0f28cca87e9afc22280c44d378d2a6e249933977" + }, + { + "limit": "2d5a6742a242091292cc0a2b607be701a45d0c4e" + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json new file mode 100644 index 000000000..e9351a1df --- /dev/null +++ b/vulnerabilities/tests/test_data/metasploit_improver/modules_metadata_base.json @@ -0,0 +1,93 @@ +{ + "auxiliary_admin/2wire/xslt_password_reset": { + "name": "2Wire Cross-Site Request Forgery Password Reset Vulnerability", + "fullname": "auxiliary/admin/2wire/xslt_password_reset", + "aliases": [ + ], + "rank": 300, + "disclosure_date": "2007-08-15", + "type": "auxiliary", + "author": [ + "hkm ", + "Travis Phillips" + ], + "description": "This module will reset the admin password on a 2Wire wireless router. This is\n done by using the /xslt page where authentication is not required, thus allowing\n configuration changes (such as resetting the password) as administrators.", + "references": [ + "CVE-2007-4387", + "OSVDB-37667", + "BID-36075", + "URL-https://seclists.org/bugtraq/2007/Aug/225" + ], + "platform": "", + "arch": "", + "rport": 80, + "autofilter_ports": [ + 80, + 8080, + 443, + 8000, + 8888, + 8880, + 8008, + 3000, + 8443 + ], + "autofilter_services": [ + "http", + "https" + ], + "targets": null, + "mod_time": "2020-10-02 17:38:06 +0000", + "path": "/modules/auxiliary/admin/2wire/xslt_password_reset.rb", + "is_install_path": true, + "ref_name": "admin/2wire/xslt_password_reset", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": false, + "needs_cleanup": false, + "actions": [ + ] + }, + "post_firefox/manage/webcam_chat": { + "name": "Firefox Webcam Chat on Privileged Javascript Shell", + "fullname": "post/firefox/manage/webcam_chat", + "aliases": [ + + ], + "rank": 300, + "disclosure_date": "2014-05-13", + "type": "post", + "author": [ + "joev " + ], + "description": "This module allows streaming a webcam from a privileged Firefox Javascript shell.", + "references": [ + "URL-http://www.rapid7.com/db/modules/exploit/firefox/local/exec_shellcode" + ], + "platform": "", + "arch": "", + "rport": null, + "autofilter_ports": null, + "autofilter_services": null, + "targets": null, + "mod_time": "2023-02-08 13:47:34 +0000", + "path": "/modules/post/firefox/manage/webcam_chat.rb", + "is_install_path": true, + "ref_name": "firefox/manage/webcam_chat", + "check": false, + "post_auth": false, + "default_credential": false, + "notes": { + }, + "session_types": [ + + ], + "needs_cleanup": null, + "actions": [ + + ] + } +} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/npm-improver-expected.json b/vulnerabilities/tests/test_data/npm/npm-improver-expected.json similarity index 100% rename from vulnerabilities/tests/test_data/npm-improver-expected.json rename to vulnerabilities/tests/test_data/npm/npm-improver-expected.json diff --git a/vulnerabilities/tests/test_data/npm_sample.json b/vulnerabilities/tests/test_data/npm/npm_sample.json similarity index 100% rename from vulnerabilities/tests/test_data/npm_sample.json rename to vulnerabilities/tests/test_data/npm/npm_sample.json diff --git a/vulnerabilities/tests/test_data/parse-advisory-npm-expected.json b/vulnerabilities/tests/test_data/npm/parse-advisory-npm-expected.json similarity index 100% rename from vulnerabilities/tests/test_data/parse-advisory-npm-expected.json rename to vulnerabilities/tests/test_data/npm/parse-advisory-npm-expected.json diff --git a/vulnerabilities/tests/test_data/package_sort/input_purls.txt b/vulnerabilities/tests/test_data/package_sort/input_purls.txt new file mode 100644 index 000000000..9f0b214b1 --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/input_purls.txt @@ -0,0 +1,103 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/actionpack@3.1.2 +pkg:gem/webbynode@1.0.5.beta10 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:github/istio/istio@0.2.2 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.10 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 diff --git a/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt b/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt new file mode 100644 index 000000000..de9405797 --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/purls_with_excel_sort.txt @@ -0,0 +1,132 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:apk/alpine/apk@2.12.9-r3?arch=x86 +pkg:apk/alpine/curl@7.83.0-r0?arch=x86 +pkg:bitbucket/birkenfeld/pygments-main@244fd47e07d1014f0aed9c +pkg:bitnami/wordpress?distro=debian-12 +pkg:bitnami/wordpress@6.2.0?arch=arm64&distro=debian-12 +pkg:bitnami/wordpress@6.2.0?arch=arm64&distro=photon-4 +pkg:bitnami/wordpress@6.2.0?distro=debian-12 +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:cocoapods/AFNetworking@4.0.1 +pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib +pkg:cocoapods/MapsIndoors@3.24.0 +pkg:cocoapods/ShareKit@2.0#Twitter +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2 +pkg:conda/openssl@1.0.2l?channel=main&subdir=linux-64&build=h077ae2c_5&type=tar.bz2 +pkg:cpan/DROLSKY/DateTime@1.55 +pkg:cpan/DROLSKY/DateTime@1.56 +pkg:cpan/DROLSKY/DateTime@1.57 +pkg:cran/caret@6.0-88 +pkg:cran/caret@6.0-89 +pkg:cran/caret@6.0-90 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:docker/cassandra@latest +pkg:docker/customer/dockerimage@sha256%3A244fd47e07d10?repository_url=gcr.io +pkg:docker/smartentry/debian@dc437cc87d10 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/actionpack@3.1.2 +pkg:gem/webbynode@1.0.5.beta10 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:github/istio/istio@0.2.2 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:hackage/Allure@0.11.0.0 +pkg:hackage/Allure@0.9.5.0 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.10 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 +pkg:swid/Acme/example.com/Enterprise+Server@1.0.0?tag_id=75b8c285-fa7b-485b-b199-4745e3004d0d +pkg:swid/Adobe+Systems+Incorporated/Adobe+InDesign@CC?tag_id=CreativeCloud-CS6-Win-GM-MUL +pkg:swid/Fedora@29?tag_id=org.fedoraproject.Fedora-29 +pkg:swift/github.com/Alamofire/Alamofire@5.4.3 +pkg:swift/github.com/RxSwiftCommunity/RxFlow@2.12.4 diff --git a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt new file mode 100644 index 000000000..886119bfd --- /dev/null +++ b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt @@ -0,0 +1,103 @@ +pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64 +pkg:alpm/arch/pacman@6.0.1-1?arch=x86_64 +pkg:alpm/arch/python-pip@21.0-1?arch=any +pkg:cargo/clap@3.0.0 +pkg:cargo/clap@3.0.1 +pkg:cargo/clap@3.0.2 +pkg:cargo/clap@3.0.10 +pkg:cargo/clap@3.0.11 +pkg:cargo/clap@3.0.20 +pkg:cargo/rand@0.7.2 +pkg:cargo/structopt@0.3.11 +pkg:composer/bk2k/bootstrap-package@7.1.0 +pkg:composer/bk2k/bootstrap-package@7.1.1 +pkg:composer/bk2k/bootstrap-package@7.1.2 +pkg:composer/bk2k/bootstrap-package@11.0.2 +pkg:composer/bk2k/bootstrap-package@11.0.3 +pkg:conan/capnproto@0.7.0 +pkg:conan/capnproto@0.8.0 +pkg:conan/capnproto@0.15.0 +pkg:conan/capnproto@0.15.2 +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch +pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch +pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed +pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 +pkg:deb/ubuntu/dpkg@1.13.21ubuntu1 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu2 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu3 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu11 +pkg:deb/ubuntu/dpkg@1.14.5ubuntu12 +pkg:gem/actionpack@3.1.1 +pkg:gem/actionpack@3.1.2 +pkg:gem/actionpack@3.1.10 +pkg:gem/actionpack@3.1.11 +pkg:gem/webbynode@1.0.5.beta2 +pkg:gem/webbynode@1.0.5.beta3 +pkg:gem/webbynode@1.0.5.beta10 +pkg:generic/postgresql@10.2.0 +pkg:generic/postgresql@10.3.0 +pkg:generic/postgresql@10.4.0 +pkg:generic/postgresql@10.19.0 +pkg:generic/postgresql@10.21.0 +pkg:generic/postgresql@10.22.0 +pkg:github/istio/istio@0.2.0 +pkg:github/istio/istio@0.2.1 +pkg:github/istio/istio@0.2.2 +pkg:github/istio/istio@0.2.10 +pkg:github/istio/istio@0.2.11 +pkg:github/istio/istio@0.2.12 +pkg:golang/github.com/1Panel-dev/1Panel@1.3.6 +pkg:golang/github.com/1Panel-dev/1Panel@1.4.3 +pkg:golang/github.com/1Panel-dev/1Panel@1.10.1-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3-lts +pkg:golang/github.com/1Panel-dev/1Panel@1.10.3 +pkg:hex/pow@1.0.2 +pkg:hex/pow@1.0.3 +pkg:hex/pow@1.0.15 +pkg:hex/pow@1.0.16 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.2.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.10.0 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2 +pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2.1 +pkg:maven/io.netty/netty-codec-dns@4.1.97.Final +pkg:maven/io.netty/netty-codec-dns@4.1.101.Final +pkg:maven/io.netty/netty-codec-http@4.1.97.Final +pkg:maven/io.netty/netty-codec-http@4.1.101.Final +pkg:maven/io.netty/netty-codec-http2@4.1.97.Final +pkg:maven/io.netty/netty-codec-http2@4.1.101.Final +pkg:npm/bootstrap-select@1.6.2 +pkg:npm/bootstrap-select@1.6.3 +pkg:npm/bootstrap-select@1.13.5 +pkg:npm/bootstrap-select@1.13.6 +pkg:npm/%40budibase/bbui@1.2.44-alpha.1 +pkg:npm/%40budibase/bbui@1.2.44-alpha.2 +pkg:npm/%40budibase/bbui@1.2.44-alpha.3 +pkg:npm/%40budibase/bbui@1.2.44-alpha.10 +pkg:npm/%40budibase/bbui@1.2.44-alpha.11 +pkg:nuget/adplug@2.3.0-beta17 +pkg:nuget/adplug@2.3.0-beta172 +pkg:nuget/adplug@2.3.0-beta173 +pkg:nuget/adplug@2.3.0-beta18 +pkg:nuget/adplug@2.3.0-beta186 +pkg:nuget/adplug@2.3.0-beta19 +pkg:nuget/adplug@2.3.0-beta190 +pkg:pypi/jinja2@2.1 +pkg:pypi/jinja2@2.1.1 +pkg:pypi/jinja2@2.2 +pkg:pypi/jinja2@2.2.1 +pkg:pypi/jinja2@2.10 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 7d8e6b258..046c86ce5 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -11,8 +11,14 @@ from django.db import connection from django.db.migrations.executor import MigrationExecutor from django.test import TestCase +from django.utils import timezone +from packageurl import PackageURL +from univers.version_range import VersionRange from vulnerabilities import severity_systems +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AffectedPackage +from vulnerabilities.importer import Reference class TestMigrations(TestCase): @@ -610,3 +616,267 @@ def setUpBeforeMigration(self, apps): def test_removal_of_duped_purls(self): Package = apps.get_model("vulnerabilities", "Package") assert Package.objects.count() == 1 + + +class TestUpdateNpmPypaAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0063_alter_packagechangelog_software_version_and_more" + migrate_to = "0064_update_npm_pypa_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="npm", name="dummy"), + affected_version_range=VersionRange.from_string("vers:npm/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + advisory_data2 = AdvisoryData( + aliases=["CVE-2020-1337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="dummy"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-1337")], + date_published=timezone.now(), + url="https://test2.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.npm.NpmImporter", + date_collected=timezone.now(), + ) + + adv2 = Advisory.objects.create( + aliases=self.advisory_data2.aliases, + summary=self.advisory_data2.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data2.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data2.references], + url=self.advisory_data2.url, + created_by="vulnerabilities.importers.pypa.PyPaImporter", + date_collected=timezone.now(), + ) + + def test_update_npm_pypa_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.pypa.PyPaImporter").count() == 0 + assert adv.filter(created_by="pypa_importer").count() == 1 + + assert adv.filter(created_by="vulnerabilities.importers.npm.NpmImporter").count() == 0 + assert adv.filter(created_by="npm_importer").count() == 1 + + +class TestUpdateNginxAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0064_update_npm_pypa_advisory_created_by" + migrate_to = "0065_update_nginx_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="nginx", name="nginx"), + affected_version_range=VersionRange.from_string("vers:nginx/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.nginx.NginxImporter", + date_collected=timezone.now(), + ) + + def test_update_nginx_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").count() == 0 + assert adv.filter(created_by="nginx_importer").count() == 1 + + +class TestUpdateGitLabAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0065_update_nginx_advisory_created_by" + migrate_to = "0066_update_gitlab_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter", + date_collected=timezone.now(), + ) + + def test_update_gitlab_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert ( + adv.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").count() == 0 + ) + assert adv.filter(created_by="gitlab_importer").count() == 1 + + +class TestUpdateGitHubAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0066_update_gitlab_advisory_created_by" + migrate_to = "0067_update_github_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.github.GitHubAPIImporter", + date_collected=timezone.now(), + ) + + def test_update_github_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert ( + adv.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").count() == 0 + ) + assert adv.filter(created_by="github_importer").count() == 1 + + +class TestUpdateNVDAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0067_update_github_advisory_created_by" + migrate_to = "0068_update_nvd_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.nvd.NVDImporter", + date_collected=timezone.now(), + ) + + def test_update_nvd_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").count() == 0 + assert adv.filter(created_by="nvd_importer").count() == 1 + + +class TestUpdatePysecAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0073_delete_packagerelatedvulnerability" + migrate_to = "0074_update_pysec_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.pysec.PyPIImporter", + date_collected=timezone.now(), + ) + + def test_update_pysec_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").count() == 0 + assert adv.filter(created_by="pysec_importer").count() == 1 diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index 7d0a5f707..40eeb6b3f 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -22,11 +22,8 @@ from vulnerabilities.importers.fireeye import FireyeImporter from vulnerabilities.importers.gentoo import GentooImporter from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.gitlab import GitLabAPIImporter from vulnerabilities.importers.istio import IstioImporter from vulnerabilities.importers.mozilla import MozillaImporter -from vulnerabilities.importers.npm import NpmImporter -from vulnerabilities.importers.pypa import PyPaImporter from vulnerabilities.importers.retiredotnet import RetireDotnetImporter from vulnerabilities.importers.ruby import RubyImporter from vulnerabilities.oval_parser import OvalParser @@ -119,12 +116,9 @@ def test_git_importer(mock_clone): ElixirSecurityImporter, FireyeImporter, GentooImporter, - GitLabAPIImporter, IstioImporter, MozillaImporter, - NpmImporter, RetireDotnetImporter, - PyPaImporter, RubyImporter, GithubOSVImporter, ], diff --git a/vulnerabilities/tests/test_debian.py b/vulnerabilities/tests/test_debian.py index ad21ef92a..25bbcb04d 100644 --- a/vulnerabilities/tests/test_debian.py +++ b/vulnerabilities/tests/test_debian.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index b8fb0935b..e6c9f9eef 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_default_improver.py b/vulnerabilities/tests/test_default_improver.py index 703e985a4..63408a522 100644 --- a/vulnerabilities/tests/test_default_improver.py +++ b/vulnerabilities/tests/test_default_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -17,6 +17,7 @@ from vulnerabilities.importer import Reference from vulnerabilities.improver import Inference from vulnerabilities.improvers.default import DefaultImprover +from vulnerabilities.improvers.default import get_exact_purls from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -136,3 +137,29 @@ def test_default_improver_with_nvd(): for data in list(default_improver.get_inferences(AdvisoryData.from_dict(advisory_data))) ] util_tests.check_results_against_json(result, expected_file) + + +def test_AffectedPackage_from_dict_should_not_crash_with_invalid_version_range(): + package = PackageURL( + type="rpm", + namespace="rpms", + name="python", + qualifiers={}, + subpath=None, + ) + + test_ranges = [ + # foo is a non-existing range + "vers:foo/1.2.3", + # apache was not supported and returned from vulnerabilities.importers.apache_httpd.ApacheHTTPDImporter + "vers:apache/", + None, + ] + for tr in test_ranges: + pkg = { + "package": package.to_dict(), + "affected_version_range": tr, + "fixed_version": None, + } + + assert AffectedPackage.from_dict(pkg) is None diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index 249181347..2531ed695 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_example.py b/vulnerabilities/tests/test_example.py index bb035b61b..119a8b61d 100644 --- a/vulnerabilities/tests/test_example.py +++ b/vulnerabilities/tests/test_example.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -85,8 +85,8 @@ def test_improve_framework_using_example_improver(self): ImproveRunner(improver_class=ExampleAliasImprover).run() assert models.Package.objects.count() == 3 - assert models.PackageRelatedVulnerability.objects.filter(fix=True).count() == 1 - assert models.PackageRelatedVulnerability.objects.filter(fix=False).count() == 2 + assert models.FixingPackageRelatedVulnerability.objects.count() == 1 + assert models.AffectedByPackageRelatedVulnerability.objects.count() == 2 assert models.VulnerabilitySeverity.objects.count() == 1 assert models.VulnerabilityReference.objects.count() == 1 diff --git a/vulnerabilities/tests/test_export.py b/vulnerabilities/tests/test_export.py index 45cdb5e5f..244940261 100644 --- a/vulnerabilities/tests/test_export.py +++ b/vulnerabilities/tests/test_export.py @@ -1,140 +1,123 @@ -import os +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + from io import StringIO from pathlib import Path from unittest import TestCase -import pytest -import saneyaml from django.core.management import call_command from django.core.management.base import CommandError +from pytest import fixture +from pytest import mark +from pytest import raises +from aboutcode import hashid +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.models import Weakness +from vulnerabilities.tests.util_tests import check_results_and_expected_files + +TEST_DATA_DIR = Path(__file__).parent / "test_data" / "export_command" + +VCID = "VCID-pst6-b358-aaap" +PURL = "pkg:generic/nginx/test@2" -@pytest.fixture +@fixture def package(db): - return Package.objects.create( - type="generic", namespace="nginx", name="test", version="2", qualifiers={}, subpath="" - ) + return Package.objects.from_purl(PURL) -@pytest.fixture +@fixture def vulnerability_reference(): - return VulnerabilityReference.objects.create( - reference_id="fake", - url=f"https://..", - ) + return VulnerabilityReference.objects.create(reference_id="fake", url=f"https://..") -@pytest.fixture +@fixture def vulnerability_severity(vulnerability_reference): return VulnerabilitySeverity.objects.create( scoring_system="cvssv3_vector", - value="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=vulnerability_reference.id, + value="7.0", + scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", + url=f"https://..", ) -@pytest.fixture +@fixture def vulnerability(db, vulnerability_reference, vulnerability_severity): - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-pst6-b358-aaap", - summary="test-vuln", - ) + vulnerability = Vulnerability.objects.create(vulnerability_id=VCID, summary="test-vuln") Alias.objects.create(alias=f"CVE-xxx-xxx-xx", vulnerability=vulnerability) VulnerabilityRelatedReference.objects.create( - reference=vulnerability_reference, vulnerability=vulnerability + reference=vulnerability_reference, + vulnerability=vulnerability, ) weakness = Weakness.objects.create(cwe_id=15) vulnerability.weaknesses.add(weakness) + vulnerability.severities.add(vulnerability_severity) return vulnerability -@pytest.fixture +@fixture def package_related_vulnerability(db, package, vulnerability): - PackageRelatedVulnerability.objects.create( + AffectedByPackageRelatedVulnerability.objects.create( package=package, vulnerability=vulnerability, - fix=False, ) return package class TestExportCommand(TestCase): def test_missing_path(self): - with pytest.raises(CommandError) as cm: + with raises(CommandError) as cm: call_command("export", stdout=StringIO()) err = str(cm) assert "Error: the following arguments are required: path" in err + @mark.django_db def test_bad_path_fail_error(self): - with pytest.raises(CommandError) as cm: + with raises(CommandError) as cm: call_command("export", "/bad path", stdout=StringIO()) err = str(cm) - assert "Please enter a valid path" in err + assert "Enter a valid directory path" in err -@pytest.mark.django_db -def test_export_data( - tmp_path, package_related_vulnerability, vulnerability_reference, vulnerability_severity +@mark.django_db +def test_run_export_command( + tmp_path, + package_related_vulnerability, + vulnerability_reference, + vulnerability_severity, ): - expected_vul = { - "vulnerability_id": "VCID-pst6-b358-aaap", - "aliases": ["CVE-xxx-xxx-xx"], - "summary": "test-vuln", - "severities": [ - { - "id": vulnerability_severity.id, - "reference_id": vulnerability_reference.id, - "scoring_system": "cvssv3_vector", - "value": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "scoring_elements": "", - "published_at": "", - } - ], - "references": [ - { - "id": vulnerability_reference.id, - "url": "https://..", - "reference_type": "", - "reference_id": "fake", - } - ], - "weaknesses": ["CWE-15"], - } - expected_pkg = { - "package": "pkg:generic/nginx/test", - "versions": [ - { - "purl": "pkg:generic/nginx/test@2", - "affected_by_vulnerabilities": ["VCID-pst6-b358-aaap"], - "fixing_vulnerabilities": [], - }, - ], - } call_command("export", tmp_path, stdout=StringIO()) - vul_filepath = os.path.join( - tmp_path, - "./aboutcode-vulnerabilities-ps/b3/VCID-pst6-b358-aaap/VCID-pst6-b358-aaap.yml", - ) - pkg_filepath = os.path.join( - tmp_path, - "./aboutcode-packages-2cf/generic/nginx/test/versions/vulnerabilities.yml", - ) + vcid_file = hashid.get_vcid_yml_file_path(vcid=VCID) + results_vuln = tmp_path / vcid_file + expected_vuln = TEST_DATA_DIR / vcid_file + check_results_and_expected_files(results_vuln, expected_vuln) + + vulns_file = hashid.get_package_vulnerabilities_yml_file_path(purl=PURL) + results_pkgvulns = tmp_path / vulns_file + expected_pkgvulns = TEST_DATA_DIR / vulns_file + check_results_and_expected_files(results_pkgvulns, expected_pkgvulns) - assert Path(vul_filepath).read_text() == saneyaml.dump(expected_vul) - assert Path(pkg_filepath).read_text() == saneyaml.dump(expected_pkg) + purls_file = hashid.get_package_purls_yml_file_path(purl=PURL) + results_pkgpurls = tmp_path / purls_file + expected_pkgpurls = TEST_DATA_DIR / purls_file + check_results_and_expected_files(results_pkgpurls, expected_pkgpurls) diff --git a/vulnerabilities/tests/test_fireeye.py b/vulnerabilities/tests/test_fireeye.py index 15935728c..f3e3bb862 100644 --- a/vulnerabilities/tests/test_fireeye.py +++ b/vulnerabilities/tests/test_fireeye.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os diff --git a/vulnerabilities/tests/test_fix_models.py b/vulnerabilities/tests/test_fix_models.py index 10845f353..cc94a41ec 100644 --- a/vulnerabilities/tests/test_fix_models.py +++ b/vulnerabilities/tests/test_fix_models.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -11,8 +11,9 @@ from django.test import TestCase from packageurl import PackageURL +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability User = get_user_model() @@ -37,20 +38,18 @@ def setUp(self): ) vuln_package = Package.objects.create(**query_kwargs) # Attaching same package to 2 vulnerabilities - PackageRelatedVulnerability.objects.create( + AffectedByPackageRelatedVulnerability.objects.create( package=vuln_package, vulnerability=vuln1, - fix=False, ) - PackageRelatedVulnerability.objects.create( + FixingPackageRelatedVulnerability.objects.create( package=vuln_package, vulnerability=vuln2, - fix=False, ) def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 20 + assert vuln_packages.count() == 10 assert vuln_packages.distinct().count() == 10 vuln_purls = [pkg.purl for pkg in vuln_packages.distinct().only(*PackageURL._fields)] assert vuln_purls == [ diff --git a/vulnerabilities/tests/test_forms.py b/vulnerabilities/tests/test_forms.py index 8999970bc..381c9c8eb 100644 --- a/vulnerabilities/tests/test_forms.py +++ b/vulnerabilities/tests/test_forms.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index 73edfb1ba..74bbaedd8 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_github_osv.py b/vulnerabilities/tests/test_github_osv.py index 559ba8d15..bcb5fdda5 100644 --- a/vulnerabilities/tests/test_github_osv.py +++ b/vulnerabilities/tests/test_github_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/tests/test_gsd.py b/vulnerabilities/tests/test_gsd.py new file mode 100644 index 000000000..41bfeff52 --- /dev/null +++ b/vulnerabilities/tests/test_gsd.py @@ -0,0 +1,240 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import datetime +import json +import os +from unittest import TestCase + +from vulnerabilities.importer import Reference +from vulnerabilities.importers.gsd import get_aliases +from vulnerabilities.importers.gsd import get_description +from vulnerabilities.importers.gsd import get_published_date_nvd_nist_gov +from vulnerabilities.importers.gsd import get_references +from vulnerabilities.importers.gsd import get_severities +from vulnerabilities.importers.gsd import get_summary +from vulnerabilities.importers.gsd import parse_advisory_data +from vulnerabilities.tests import util_tests + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DATA = os.path.join(BASE_DIR, "test_data/gsd") + + +class TestGSDImporter(TestCase): + def test_to_advisories1(self): + with open(os.path.join(TEST_DATA, "GSD-2016-20005.json")) as f: + raw_data = json.load(f) + imported_data = parse_advisory_data(raw_data, "GSD-2016-20005.json") + expected_file = os.path.join(TEST_DATA, "GSD-2016-20005-expected.json") + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_to_advisories2(self): + with open(os.path.join(TEST_DATA, "GSD-2022-4030.json")) as f: + raw_data = json.load(f) + imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") + expected_file = os.path.join(TEST_DATA, "GSD-2022-4030-expected.json") + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_to_advisories3(self): + with open(os.path.join(TEST_DATA, "GSD-2002-0001.json")) as f: + raw_data = json.load(f) + imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") + expected_file = os.path.join(TEST_DATA, "GSD-2002-0001-expected.json") + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_to_advisories4(self): + with open(os.path.join(TEST_DATA, "GSD-2006-0326.json")) as f: + raw_data = json.load(f) + imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") + expected_file = os.path.join(TEST_DATA, "GSD-2006-0326-expected.json") + result = imported_data.to_dict() + util_tests.check_results_against_json(result, expected_file) + + def test_get_references(self): + assert get_references( + { + "references": { + "reference_data": [ + { + "name": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", + "refsource": "CONFIRM", + "tags": ["Vendor Advisory"], + "url": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", + } + ] + } + } + ) == [ + Reference( + reference_id="", + url="https://kc.mcafee.com/corporate/index?page=content&id=SB10198", + severities=[], + ) + ] + + def test_get_description(self): + assert get_description( + { + "description": { + "description_data": [ + { + "lang": "eng", + "value": "User Name Disclosure in the server in McAfee Network Data Loss Prevention (NDLP) 9.3.x allows remote attackers to view user information via the appliance web interface.", + } + ] + } + } + ) == [ + "User Name Disclosure in the server in McAfee Network Data Loss Prevention (NDLP) 9.3.x allows remote attackers to view user information via the appliance web interface." + ] + + def test_get_aliases_cve_org(self): + assert get_aliases( + { + "CVE_data_meta": { + "ASSIGNER": "secure@intel.com", + "ID": "CVE-2017-4017", + "STATE": "PUBLIC", + } + } + ) == ["CVE-2017-4017"] + assert get_aliases( + { + "CVE_data_meta": { + "ASSIGNER": "secure@intel.com", + "ID": "CVE-2017-4017", + "STATE": "PUBLIC", + }, + "source": {"advisory": "GHSA-v8x6-59g4-5g3w", "discovery": "UNKNOWN"}, + } + ) == ["CVE-2017-4017", "GHSA-v8x6-59g4-5g3w"] + assert get_aliases( + {"source": {"advisory": "GHSA-v8x6-59g4-5g3w", "discovery": "UNKNOWN"}} + ) == ["GHSA-v8x6-59g4-5g3w"] + + def test_get_summary(self): + assert ( + get_summary({"CVE_data_meta": {"TITLE": "DoS vulnerability: Invalid Accent Colors"}}) + == "DoS vulnerability: Invalid Accent Colors" + ) + + def test_get_severities(self): + assert get_severities( + { + "impact": { + "cvss": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 5.7, + "baseSeverity": "MEDIUM", + "confidentialityImpact": "NONE", + "integrityImpact": "NONE", + "privilegesRequired": "LOW", + "scope": "UNCHANGED", + "userInteraction": "REQUIRED", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H", + "version": "3.1", + } + } + } + ) == ["CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H"] + assert get_severities( + { + "impact": { + "baseMetricV2": { + "acInsufInfo": False, + "cvssV2": { + "accessComplexity": "LOW", + "accessVector": "NETWORK", + "authentication": "NONE", + "availabilityImpact": "PARTIAL", + "baseScore": 7.5, + "confidentialityImpact": "PARTIAL", + "integrityImpact": "PARTIAL", + "vectorString": "AV:N/AC:L/Au:N/C:P/I:P/A:P", + "version": "2.0", + }, + "exploitabilityScore": 10.0, + "impactScore": 6.4, + "obtainAllPrivilege": False, + "obtainOtherPrivilege": False, + "obtainUserPrivilege": False, + "severity": "HIGH", + "userInteractionRequired": False, + }, + "baseMetricV3": { + "cvssV3": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 9.8, + "baseSeverity": "CRITICAL", + "confidentialityImpact": "HIGH", + "integrityImpact": "HIGH", + "privilegesRequired": "NONE", + "scope": "UNCHANGED", + "userInteraction": "NONE", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "version": "3.1", + }, + "exploitabilityScore": 3.9, + "impactScore": 5.9, + }, + } + } + ) == ["AV:N/AC:L/Au:N/C:P/I:P/A:P", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"] + + assert get_severities( + { + "impact": { + "cvss": [ + { + "baseScore": 8.1, + "baseSeverity": "HIGH", + "vectorString": "CVSS:3.1/A:H/I:H/C:N/S:U/UI:N/PR:L/AC:L/AV:N", + "version": "3.1", + } + ] + } + } + ) == ["CVSS:3.1/A:H/I:H/C:N/S:U/UI:N/PR:L/AC:L/AV:N"] + + assert get_severities( + { + "impact": { + "baseMetricV3": { + "cvssV3": { + "attackComplexity": "LOW", + "attackVector": "NETWORK", + "availabilityImpact": "HIGH", + "baseScore": 8.1, + "baseSeverity": "HIGH", + "confidentialityImpact": "NONE", + "integrityImpact": "HIGH", + "privilegesRequired": "LOW", + "scope": "UNCHANGED", + "userInteraction": "NONE", + "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:H", + "version": "3.1", + }, + "exploitabilityScore": 2.8, + "impactScore": 5.2, + } + } + } + ) == ["CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:H"] + + def test_get_published_date_nvd_nist_gov(self): + assert get_published_date_nvd_nist_gov( + {"publishedDate": "2022-06-23T07:15Z"} + ) == datetime.datetime(2022, 6, 23, 7, 15, 0, 0).replace(tzinfo=datetime.timezone.utc) + assert get_published_date_nvd_nist_gov({}) is None diff --git a/vulnerabilities/tests/test_import_command.py b/vulnerabilities/tests/test_import_command.py index e2f5ac9a0..5734a7942 100644 --- a/vulnerabilities/tests/test_import_command.py +++ b/vulnerabilities/tests/test_import_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 123011266..3f88e0963 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_importer.py b/vulnerabilities/tests/test_importer.py index 7298bc4b6..ebfc8592c 100644 --- a/vulnerabilities/tests/test_importer.py +++ b/vulnerabilities/tests/test_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_improve_command.py b/vulnerabilities/tests/test_improve_command.py index 03b74f3b1..b6b152b40 100644 --- a/vulnerabilities/tests/test_improve_command.py +++ b/vulnerabilities/tests/test_improve_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_improve_runner.py b/vulnerabilities/tests/test_improve_runner.py index e4b820c33..347f87c97 100644 --- a/vulnerabilities/tests/test_improve_runner.py +++ b/vulnerabilities/tests/test_improve_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -24,9 +24,10 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference diff --git a/vulnerabilities/tests/test_improver.py b/vulnerabilities/tests/test_improver.py index 8c05223c2..e526d4ac9 100644 --- a/vulnerabilities/tests/test_improver.py +++ b/vulnerabilities/tests/test_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index c3c715e5a..706de1422 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 2efe45e86..014754786 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -51,12 +51,10 @@ def test_package_to_vulnerability(self): p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - prv1 = models.PackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1, fix=False - ) - prv2 = models.PackageRelatedVulnerability.objects.create( - package=p2, vulnerability=v1, fix=True + prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( + package=p1, vulnerability=v1 ) + prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) assert p1.fixing_vulnerabilities.count() == 0 @@ -68,12 +66,10 @@ def test_vulnerability_package(self): p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - prv1 = models.PackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1, fix=False - ) - prv2 = models.PackageRelatedVulnerability.objects.create( - package=p2, vulnerability=v1, fix=True + prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( + package=p1, vulnerability=v1 ) + prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) assert v1.vulnerable_packages.count() == 1 assert v1.fixed_by_packages.count() == 1 @@ -110,10 +106,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_1_1, vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - fix=False, ) # aliases @@ -133,10 +128,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.FixingPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_3_6, vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - fix=True, ) # vuln for fixed pkg -- and also vuln # 2 for affected pkg @@ -146,10 +140,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_3_6, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=False, ) # aliases @@ -161,10 +154,9 @@ def setUp(self): # vuln # 2 for affected pkg -- already defined above bc also vuln for fixed pkg above! # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_1_1, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=False, ) # aliases -- already defined above @@ -180,10 +172,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.FixingPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_5_0_0b1, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=True, ) # This vulnerability does not affect any redis packages in this set of tests but does affect a made-up package, self.package_pypi_bogus_1_2_3. @@ -203,10 +194,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_bogus_1_2_3, vulnerability=self.vuln_VCID_abcd_efgh_1234, - fix=False, ) # This vulnerability does not affect any packages in this set of tests included to test .all(). @@ -433,8 +423,11 @@ def test_sort_by_version(self): version="3.0.0", ) - sorted_pkgs = requesting_package.sort_by_version(vuln_pkg_list) - first_sorted_item = sorted_pkgs[0] + requesting_package.calculate_version_rank + + sorted_pkgs = Package.objects.filter(package_url__in=list_to_sort) + + sorted_pkgs = list(sorted_pkgs) assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" @@ -579,47 +572,18 @@ def test_get_affecting_vulnerabilities_package_method(self): assert redis_4_1_1_affecting_vulnerabilities == affecting_vulnerabilities def test_get_non_vulnerable_versions(self): - """ - Return a tuple of the next and latest non-vulnerable versions of this package as PackageURLs. - """ - searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 - redis_4_1_1_non_vulnerable_versions = ( - searched_for_package_redis_4_1_1.get_non_vulnerable_versions() - ) - - non_vulnerable_versions = ( - PackageURL( - type="pypi", - namespace=None, - name="redis", - version="5.0.0b1", - qualifiers={}, - subpath=None, - ), - PackageURL( - type="pypi", - namespace=None, - name="redis", - version="5.0.0b1", - qualifiers={}, - subpath=None, - ), - ) - - assert redis_4_1_1_non_vulnerable_versions == non_vulnerable_versions + redis_next, redis_later = self.package_pypi_redis_4_1_1.get_non_vulnerable_versions() + assert redis_next.version == "5.0.0b1" + assert redis_later.version == "5.0.0b1" def test_version_class_and_current_version(self): - searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 + package = self.package_pypi_redis_4_1_1 - package_version_class = RANGE_CLASS_BY_SCHEMES[ - searched_for_package_redis_4_1_1.type - ].version_class + package_version_class = RANGE_CLASS_BY_SCHEMES[package.type].version_class assert package_version_class == versions.PypiVersion - assert searched_for_package_redis_4_1_1.current_version == package_version_class( - string="4.1.1" - ) - assert str(searched_for_package_redis_4_1_1.current_version) == "4.1.1" + assert package.current_version == package_version_class(string="4.1.1") + assert str(package.current_version) == "4.1.1" def test_get_fixed_by_package_versions(self): searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index 8086f0183..15437e22b 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_msr2019.py b/vulnerabilities/tests/test_msr2019.py index 4af8334e3..16696dd71 100644 --- a/vulnerabilities/tests/test_msr2019.py +++ b/vulnerabilities/tests/test_msr2019.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_openssl.py b/vulnerabilities/tests/test_openssl.py index 7bbeb895e..0effc9515 100644 --- a/vulnerabilities/tests/test_openssl.py +++ b/vulnerabilities/tests/test_openssl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_oss_fuzz.py b/vulnerabilities/tests/test_oss_fuzz.py index d44c7ab9c..27f5ebfc4 100644 --- a/vulnerabilities/tests/test_oss_fuzz.py +++ b/vulnerabilities/tests/test_oss_fuzz.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os diff --git a/vulnerabilities/tests/test_osv.py b/vulnerabilities/tests/test_osv.py index 5779d0589..e7505510c 100644 --- a/vulnerabilities/tests/test_osv.py +++ b/vulnerabilities/tests/test_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import datetime diff --git a/vulnerabilities/tests/test_performance.py b/vulnerabilities/tests/test_performance.py index 05f043aed..ba17bfaa9 100644 --- a/vulnerabilities/tests/test_performance.py +++ b/vulnerabilities/tests/test_performance.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import pytest diff --git a/vulnerabilities/tests/test_postgres_workaround.py b/vulnerabilities/tests/test_postgres_workaround.py index 0da44e256..3b0f215ab 100644 --- a/vulnerabilities/tests/test_postgres_workaround.py +++ b/vulnerabilities/tests/test_postgres_workaround.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_postgresql.py b/vulnerabilities/tests/test_postgresql.py index b53d55df3..0e9e71481 100644 --- a/vulnerabilities/tests/test_postgresql.py +++ b/vulnerabilities/tests/test_postgresql.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index 7a16e49c1..f5be28dca 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index 2fa9c0669..52070b2d1 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py new file mode 100644 index 000000000..420c8c402 --- /dev/null +++ b/vulnerabilities/tests/test_risk.py @@ -0,0 +1,185 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest + +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness +from vulnerabilities.risk import compute_vulnerability_risk_factors +from vulnerabilities.risk import get_exploitability_level +from vulnerabilities.risk import get_weighted_severity +from vulnerabilities.severity_systems import CVSSV3 +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.severity_systems import GENERIC + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability(): + vul = Vulnerability(vulnerability_id="VCID-Existing") + vul.save() + + severity1 = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", + scoring_system=CVSSV3.identifier, + scoring_elements="CVSS:3.0/AV:P/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:N/E:H/RL:O/RC:R/CR:H/MAC:H/MC:L", + value="6.5", + ) + + severity2 = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", + scoring_system=GENERIC.identifier, + value="MODERATE", # 6.9 + ) + vul.severities.add(severity1) + vul.severities.add(severity2) + + weaknesses = Weakness.objects.create(cwe_id=119) + vul.weaknesses.add(weaknesses) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def exploit(): + vul = Vulnerability(vulnerability_id="VCID-Exploit") + vul.save() + return Exploit.objects.create(vulnerability=vul, description="exploit description") + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability_with_exploit_ref(): + vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") + vul.save() + + reference_exploit = VulnerabilityReference.objects.create( + reference_id="", + reference_type=VulnerabilityReference.EXPLOIT, + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxxx2", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference_exploit, vulnerability=vul) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def high_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") + vul.save() + + severity = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx3", + scoring_system=EPSS.identifier, + value=".9", + ) + vul.severities.add(severity) + + return vul.severities.all() + + +@pytest.fixture +@pytest.mark.django_db +def low_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") + vul.save() + + severity = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx4", + scoring_system=EPSS.identifier, + value=".3", + ) + vul.severities.add(severity) + + return vul.severities.all() + + +@pytest.mark.django_db +def test_exploitability_level( + exploit, vulnerability_with_exploit_ref, high_epss_score, low_epss_score +): + + assert get_exploitability_level(exploit, None, None) == 2 + + assert get_exploitability_level(None, None, high_epss_score) == 2 + + assert get_exploitability_level(None, None, low_epss_score) == 0.5 + + assert ( + get_exploitability_level( + exploits=None, + references=vulnerability_with_exploit_ref.references.all(), + severities=vulnerability_with_exploit_ref.severities.all(), + ) + == 1 + ) + + assert get_exploitability_level(None, None, None) == 0.5 + + +@pytest.mark.django_db +def test_get_weighted_severity(vulnerability): + severities = vulnerability.severities.all() + assert get_weighted_severity(severities) == 6.2 + + severity2 = VulnerabilitySeverity.objects.create( + url="https://security-tracker.debian.org/tracker/CVE-2019-13057", + scoring_system=GENERIC.identifier, + value="CRITICAL", + ) + vulnerability.severities.add(severity2) + + new_severities = vulnerability.severities.all() + assert get_weighted_severity(new_severities) == 7 + + +@pytest.mark.django_db +def test_compute_vulnerability_risk_factors(vulnerability, exploit): + severities = vulnerability.severities.all() + references = vulnerability.references.all() + + assert compute_vulnerability_risk_factors(references, severities, exploit) == ( + 6.2, + 2, + ) + + assert compute_vulnerability_risk_factors(references, severities, None) == (6.2, 0.5) + + assert compute_vulnerability_risk_factors(references, None, exploit) == (0, 2) + + assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) + + +@pytest.mark.django_db +def test_get_vulnerability_risk_score(vulnerability): + vulnerability.weighted_severity = 6.0 + vulnerability.exploitability = 2 + + assert vulnerability.risk_score == 10.0 # max risk_score can be reached + + vulnerability.weighted_severity = 6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == 3.0 + + vulnerability.weighted_severity = 5.6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == 2.8 + + vulnerability.weighted_severity = None + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score is None + + vulnerability.weighted_severity = None + vulnerability.exploitability = None + assert vulnerability.risk_score is None diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index 0e06afe1d..e66300512 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 8a6d88e47..58b7c4302 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py index 5b2691b43..78a59ecc8 100644 --- a/vulnerabilities/tests/test_suse.py +++ b/vulnerabilities/tests/test_suse.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_backports.py b/vulnerabilities/tests/test_suse_backports.py index 6253b7018..aa3737246 100644 --- a/vulnerabilities/tests/test_suse_backports.py +++ b/vulnerabilities/tests/test_suse_backports.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_oval.py b/vulnerabilities/tests/test_suse_oval.py index eba1ca0f2..436cfcf49 100644 --- a/vulnerabilities/tests/test_suse_oval.py +++ b/vulnerabilities/tests/test_suse_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_scores.py b/vulnerabilities/tests/test_suse_scores.py index 0abdd48b1..96a5d0d07 100644 --- a/vulnerabilities/tests/test_suse_scores.py +++ b/vulnerabilities/tests/test_suse_scores.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_throttling.py b/vulnerabilities/tests/test_throttling.py index 364ef487e..dbf8d759d 100644 --- a/vulnerabilities/tests/test_throttling.py +++ b/vulnerabilities/tests/test_throttling.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index fa54fa1af..b31588347 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index cac58e47a..d28d036b7 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py index 925d28d80..71b6d86af 100644 --- a/vulnerabilities/tests/test_upstream.py +++ b/vulnerabilities/tests/test_upstream.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -19,7 +19,7 @@ ) def test_updated_advisories(importer_name, importer_class): # FIXME: why are we doing this? - if importer_name.endswith("GitHubAPIImporter"): + if importer_name.endswith("GitHubAPIImporterPipeline"): return advisory_datas = importer_class().advisory_data() diff --git a/vulnerabilities/tests/test_utils.py b/vulnerabilities/tests/test_utils.py index e67aa18bf..c9ba98e79 100644 --- a/vulnerabilities/tests/test_utils.py +++ b/vulnerabilities/tests/test_utils.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 20eb880af..692305f8d 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -3,19 +3,29 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +import os + +import pytest from django.test import Client from django.test import TestCase from packageurl import PackageURL +from univers import versions from vulnerabilities.models import Alias from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability +from vulnerabilities.templatetags.url_filters import url_quote_filter from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch +from vulnerabilities.views import get_purl_version_class +from vulnerabilities.views import purl_sort_key + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +TEST_DIR = os.path.join(BASE_DIR, "test_data/package_sort") class PackageSearchTestCase(TestCase): @@ -176,3 +186,90 @@ def test_robots_txt(self): assert response.status_code == 200 response = self.client.post("/robots.txt") assert response.status_code == 405 + + +class TestPackageSortTestCase(TestCase): + def setUp(self): + self.client = Client() + TEST_DATA = os.path.join(TEST_DIR, "input_purls.txt") + with open(TEST_DATA) as f: + input_purls = [l for l in f.readlines()] + self.input_purls = input_purls + for pkg in input_purls: + real_purl = PackageURL.from_string(pkg) + attrs = {k: v for k, v in real_purl.to_dict().items() if v} + Package.objects.create(**attrs) + + def test_sorted_queryset(self): + qs_all = Package.objects.all() + pkgs_qs_all = list(qs_all) + sorted_pkgs_qs_all = sorted(pkgs_qs_all, key=purl_sort_key) + + pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] + sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") + with open(sorted_purls, "r") as f: + expected_content = f.read().splitlines() + assert pkg_package_urls == expected_content + + def test_get_purl_version_class(self): + test_cases = { + "pkg:alpm/arch/containers-common@1:0.47.4-4?arch=x86_64": versions.ArchLinuxVersion, + "pkg:cargo/clap@3.0.0": versions.SemverVersion, + "pkg:composer/bk2k/bootstrap-package@7.1.0": versions.ComposerVersion, + "pkg:conan/capnproto@0.7.0": versions.ConanVersion, + "pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch": versions.DebianVersion, + "pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed": versions.DebianVersion, + "pkg:gem/actionpack@3.1.1": versions.RubygemsVersion, + "pkg:generic/postgresql@10.2.0": versions.SemverVersion, + "pkg:github/istio/istio@0.2.0": versions.SemverVersion, + "pkg:golang/github.com/1Panel-dev/1Panel@1.3.6": versions.GolangVersion, + "pkg:hex/pow@1.0.2": versions.SemverVersion, + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.1.1": versions.MavenVersion, + "pkg:npm/bootstrap-select@1.6.2": versions.SemverVersion, + "pkg:nuget/adplug@2.3.0-beta17": versions.NugetVersion, + "pkg:pypi/jinja2@2.1": versions.PypiVersion, + "pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11": versions.RpmVersion, + } + for k in test_cases: + pkg = Package.objects.get(package_url=k) + assert get_purl_version_class(pkg) == test_cases.get(k) + + +class TestCustomFilters: + @pytest.mark.parametrize( + "input_value, expected_output", + [ + ( + "pkg:rpm/redhat/katello-client-bootstrap@1.1.0-2?arch=el6sat", + "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat", + ), + ( + "pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", + "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain", + ), + ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), + ( + "pkg:deb/ubuntu/nginx@0.6.34-2ubuntu1~intrepid1", + "pkg%3Adeb/ubuntu/nginx%400.6.34-2ubuntu1~intrepid1", + ), + ( + "pkg:rpm/redhat/openssl@1:1.0.2k-16.el7_6?arch=1", + "pkg%3Arpm/redhat/openssl%401%3A1.0.2k-16.el7_6%3Farch%3D1", + ), + ( + "pkg:golang/google.golang.org/genproto#googleapis/api/annotations", + "pkg%3Agolang/google.golang.org/genproto%23googleapis/api/annotations", + ), + ( + "pkg:cocoapods/GoogleUtilities@7.5.2#NSData+zlib", + "pkg%3Acocoapods/GoogleUtilities%407.5.2%23NSData%2Bzlib", + ), + ( + "pkg:conda/absl-py@0.4.1?build=py36h06a4308_0&channel=main&subdir=linux-64&type=tar.bz2", + "pkg%3Aconda/absl-py%400.4.1%3Fbuild%3Dpy36h06a4308_0%26channel%3Dmain%26subdir%3Dlinux-64%26type%3Dtar.bz2", + ), + ], + ) + def test_url_quote_filter(self, input_value, expected_output): + filtered = url_quote_filter(input_value) + assert filtered == expected_output diff --git a/vulnerabilities/tests/test_vulnerability_status_improver.py b/vulnerabilities/tests/test_vulnerability_status_improver.py index 5bad2f498..89a57c0b7 100644 --- a/vulnerabilities/tests/test_vulnerability_status_improver.py +++ b/vulnerabilities/tests/test_vulnerability_status_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,13 +13,12 @@ import pytest -from vulnerabilities.importers.nvd import NVDImporter from vulnerabilities.improvers.vulnerability_status import VulnerabilityStatusImprover -from vulnerabilities.improvers.vulnerability_status import get_status_from_api from vulnerabilities.models import Advisory from vulnerabilities.models import Alias from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityStatusType +from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -34,13 +33,13 @@ def test_interesting_advisories(): Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="1", date_collected=datetime.now(), ) Advisory.objects.create( aliases=["CVE-1"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="2", date_collected=datetime.now(), ) @@ -55,7 +54,7 @@ def test_improver_end_to_end(mock_response): mock_response.return_value = response adv = Advisory.objects.create( aliases=["CVE-2023-35866"], - created_by=NVDImporter.qualified_name, + created_by=NVDImporterPipeline.pipeline_id, summary="1", date_collected=datetime.now(), ) diff --git a/vulnerabilities/tests/test_xen.py b/vulnerabilities/tests/test_xen.py index d9e913f68..3e5822b1e 100644 --- a/vulnerabilities/tests/test_xen.py +++ b/vulnerabilities/tests/test_xen.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/util_tests.py b/vulnerabilities/tests/util_tests.py index b70c6381f..5690fad4b 100644 --- a/vulnerabilities/tests/util_tests.py +++ b/vulnerabilities/tests/util_tests.py @@ -3,12 +3,13 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json import os +from pathlib import Path import saneyaml @@ -34,21 +35,19 @@ def check_results_against_json( If ``regen`` is True, the ``expected_file`` is overwritten with the ``results`` data. This is convenient for updating tests expectations. """ + expected_file = Path(expected_file) if regen: - with open(expected_file, "w") as reg: - json.dump(results, reg, indent=2, separators=(",", ": ")) + exp = json.dumps(results, indent=2, separators=(",", ": ")) + expected_file.write_text(exp) expected = results else: - with open(expected_file) as exp: - expected = json.load(exp) + exp = expected_file.read_text() + expected = json.loads(exp) check_results_against_expected(results, expected) -def check_results_against_expected( - results, - expected, -): +def check_results_against_expected(results, expected): """ Check the JSON-serializable mapping or sequence ``results`` against the ``expected``. @@ -57,3 +56,24 @@ def check_results_against_expected( # the failures comparison/diff if results != expected: assert saneyaml.dump(results) == saneyaml.dump(expected) + + +def check_results_and_expected_files( + results_file, + expected_file, + regen=VULNERABLECODE_REGEN_TEST_FIXTURES, +): + """ + Check the text content of a results_files and an expected_file. + + If ``regen`` is True, the ``expected_file`` is overwritten with the + ``results_file`` content. This is convenient for updating tests expectations. + """ + results = results_file.read_text() + if regen: + expected_file.parent.mkdir(parents=True, exist_ok=True) + expected_file.write_text(results) + expected = results + else: + expected = expected_file.read_text() + assert results == expected diff --git a/vulnerabilities/throttling.py b/vulnerabilities/throttling.py index d439a04b2..99b1d7756 100644 --- a/vulnerabilities/throttling.py +++ b/vulnerabilities/throttling.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from rest_framework.exceptions import Throttled diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index c6874b7df..969a08f2f 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -3,14 +3,13 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import bisect import csv import dataclasses -import datetime import json import logging import os @@ -18,7 +17,6 @@ import urllib.request from collections import defaultdict from functools import total_ordering -from hashlib import sha256 from http import HTTPStatus from typing import List from typing import Optional @@ -26,23 +24,22 @@ from typing import Union from unittest.mock import MagicMock from urllib.parse import urljoin -from uuid import uuid4 import requests import saneyaml import toml import urllib3 from packageurl import PackageURL -from packageurl import normalize_qualifiers -from packageurl import normalize_subpath -from packageurl.contrib.django.models import without_empty_values +from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import NginxVersionRange from univers.version_range import VersionRange +from aboutcode.hashid import build_vcid # NOQA + logger = logging.getLogger(__name__) -cve_regex = re.compile(r"CVE-\d{4}-\d{4,7}", re.IGNORECASE) +cve_regex = re.compile(r"CVE-[0-9]{4}-[0-9]{4,19}", re.IGNORECASE) is_cve = cve_regex.match find_all_cve = cve_regex.findall @@ -360,63 +357,6 @@ def resolve_version_range( return affected_versions, unaffected_versions -def build_vcid(prefix="VCID"): - """ - Return a new VulnerableCode VCID unique identifier string using the ``prefix``. - - For example:: - >>> import re - >>> vcid = build_vcid() - >>> # VCID-6npv-94wz-hhuq - >>> assert re.match('VCID(-[a-z1-9]{4}){3}', vcid), vcid - """ - # we keep only 64 bits (e.g. 8 bytes) - uid = sha256(uuid4().bytes).digest()[:8] - # we keep only 12 encoded bytes (which corresponds to 60 bits) - uid = base32_custom(uid)[:12].decode("utf-8").lower() - return f"{prefix}-{uid[:4]}-{uid[4:8]}-{uid[8:12]}" - - -_base32_alphabet = b"ABCDEFGHJKMNPQRSTUVWXYZ123456789" -_base32_table = None - - -def base32_custom(btes): - """ - Encode the ``btes`` bytes object using a Base32 encoding using a custom - alphabet and return a bytes object. - - Code copied and modified from the Python Standard Library: - base64.b32encode function - - SPDX-License-Identifier: Python-2.0 - Copyright (c) The Python Software Foundation - - For example:: - >>> assert base32_custom(b'abcd') == b'ABTZE25E', base32_custom(b'abcd') - >>> assert base32_custom(b'abcde00000xxxxxPPPPP') == b'PFUGG3DFGA2DAPBTSB6HT8D2MBJFAXCT' - """ - global _base32_table - # Delay the initialization of the table to not waste memory - # if the function is never called - if _base32_table is None: - b32tab = [bytes((i,)) for i in _base32_alphabet] - _base32_table = [a + b for a in b32tab for b in b32tab] - - encoded = bytearray() - from_bytes = int.from_bytes - - for i in range(0, len(btes), 5): - c = from_bytes(btes[i : i + 5], "big") - encoded += ( - _base32_table[c >> 30] - + _base32_table[(c >> 20) & 0x3FF] # bits 1 - 10 - + _base32_table[(c >> 10) & 0x3FF] # bits 11 - 20 - + _base32_table[c & 0x3FF] # bits 21 - 30 # bits 31 - 40 - ) - return bytes(encoded) - - def fetch_response(url): """ Fetch and return `response` from the `url` @@ -559,6 +499,9 @@ def get_importer_name(advisory): def get_advisory_url(file, base_path, url): + """ + Return the advisory URL constructed by combining the base URL with the relative file path. + """ relative_path = str(file.relative_to(base_path)).strip("/") advisory_url = urljoin(url, relative_path) return advisory_url diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 88128b509..fd57acea5 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging @@ -23,6 +23,8 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -37,6 +39,29 @@ PAGE_SIZE = 20 +def purl_sort_key(purl: models.Package): + """ + Return a sort key for the built-in sorted() function when sorting a list + of Package objects. If the Package ``type`` is supported by univers, apply + the univers version class to the Package ``version``, and otherwise use the + ``version`` attribute as is. + """ + purl_version_class = get_purl_version_class(purl) + purl_sort_version = purl.version + if purl_version_class: + purl_sort_version = purl_version_class(purl.version) + return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) + + +def get_purl_version_class(purl: models.Package): + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class + return purl_version_class + + class PackageSearch(ListView): model = models.Package template_name = "packages.html" @@ -94,7 +119,10 @@ def get_context_data(self, **kwargs): package = self.object context["package"] = package context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") - context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") + # Ghost package should not fix any vulnerability. + context["fixing_vulnerabilities"] = ( + None if package.is_ghost else package.fixing.order_by("vulnerability_id") + ) context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details @@ -128,7 +156,17 @@ class VulnerabilityDetails(DetailView): slug_field = "vulnerability_id" def get_queryset(self): - return super().get_queryset().prefetch_related("references", "aliases", "weaknesses") + return ( + super() + .get_queryset() + .prefetch_related( + "references", + "aliases", + "weaknesses", + "severities", + "exploits", + ) + ) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -140,7 +178,7 @@ def get_context_data(self, **kwargs): severity_vectors = [] severity_values = set() - for s in self.object.severities: + for s in self.object.severities.all(): if s.scoring_system == EPSS.identifier: continue @@ -159,20 +197,52 @@ def get_context_data(self, **kwargs): if s.value: severity_values.add(s.value) + sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) + sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) + + all_affected_fixed_by_matches = [] + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = {} + affected_fixed_by_matches["affected_package"] = sorted_affected_package + matched_fixed_by_packages = [] + for fixed_by_package in sorted_fixed_by_packages: + + # Ghost Package can't fix vulnerability. + if fixed_by_package.is_ghost: + continue + + sorted_affected_version_class = get_purl_version_class(sorted_affected_package) + fixed_by_version_class = get_purl_version_class(fixed_by_package) + if ( + (fixed_by_package.type == sorted_affected_package.type) + and (fixed_by_package.namespace == sorted_affected_package.namespace) + and (fixed_by_package.name == sorted_affected_package.name) + and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) + and (fixed_by_package.subpath == sorted_affected_package.subpath) + and ( + fixed_by_version_class(fixed_by_package.version) + > sorted_affected_version_class(sorted_affected_package.version) + ) + ): + matched_fixed_by_packages.append(fixed_by_package.purl) + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + context.update( { "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(self.object.severities), + "severities": list(self.object.severities.all()), "severity_score_range": get_severity_range(severity_values), "severity_vectors": severity_vectors, "references": self.object.references.all(), "aliases": self.object.aliases.all(), - "affected_packages": self.object.affected_packages.all(), - "fixed_by_packages": self.object.fixed_by_packages.all(), + "affected_packages": sorted_affected_packages, + "fixed_by_packages": sorted_fixed_by_packages, "weaknesses": weaknesses_present_in_db, "status": status, "history": self.object.history, + "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) return context diff --git a/vulnerabilities/weight_config.py b/vulnerabilities/weight_config.py new file mode 100644 index 000000000..f9ad8292d --- /dev/null +++ b/vulnerabilities/weight_config.py @@ -0,0 +1,2919 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +WEIGHT_CONFIG = { + "nvd.nist.gov": 9, + "api.first.org": 9, + "github.com": 9, + "access.redhat.com": 9, + "bugzilla.redhat.com": 9, + "cve.mitre.org": 9, + "people.canonical.com": 9, + "lists.apache.org": 9, + "ubuntu.com": 9, + "openwall.com": 9, + "lists.fedoraproject.org": 9, + "npmjs.com": 9, + "web.archive.org": 9, + "security.archlinux.org": 9, + "mozilla.org": 9, + "huntr.dev": 9, + "snyk.io": 9, + "vuldb.com": 9, + "jenkins.io": 9, + "security.netapp.com": 9, + "securityfocus.com": 9, + "lists.debian.org": 9, + "rustsec.org": 8, + "usn.ubuntu.com": 8, + "packetstormsecurity.com": 8, + "pkg.go.dev": 8, + "hackerone.com": 8, + "git.kernel.org": 8, + "groups.google.com": 8, + "security.snyk.io": 8, + "lists.opensuse.org": 8, + "rhn.redhat.com": 8, + "seclists.org": 8, + "code.google.com": 8, + "moodle.org": 8, + "portal.msrc.microsoft.com": 8, + "issues.apache.org": 7, + "debian.org": 7, + "jira.xwiki.org": 7, + "exchange.xforce.ibmcloud.com": 7, + "typo3.org": 7, + "exploit-db.com": 7, + "gist.github.com": 7, + "plugins.trac.wordpress.org": 7, + "bugs.wireshark.org": 7, + "git.moodle.org": 7, + "secunia.com": 7, + "huntr.com": 7, + "security.gentoo.org": 7, + "bugs.launchpad.net": 7, + "code.wireshark.org": 7, + "svn.apache.org": 7, + "httpd.apache.org": 7, + "securitytracker.com": 7, + "wordfence.com": 7, + "wireshark.org": 7, + "bugs.chromium.org": 7, + "gitlab.com": 7, + "silverstripe.org": 7, + "crbug.com": 7, + "phpmyadmin.net": 7, + "securitylab.github.com": 7, + "curl.se": 7, + "sourceforge.net": 7, + "drupal.org": 7, + "discuss.hashicorp.com": 7, + "pypi.org": 7, + "djangoproject.com": 7, + "bugzilla.mozilla.org": 7, + "raw.githubusercontent.com": 7, + "lists.gnu.org": 7, + "bugs.debian.org": 7, + "security-tracker.debian.org": 7, + "marc.info": 7, + "support.apple.com": 7, + "blogs.gentoo.org": 7, + "bitbucket.org": 7, + "bugzilla.suse.com": 7, + "go.dev": 7, + "pivotal.io": 7, + "whitesourcesoftware.com": 7, + "cdn.kernel.org": 7, + "gitee.com": 7, + "lore.kernel.org": 7, + "ruby-lang.org": 7, + "msrc.microsoft.com": 7, + "wordpress.org": 7, + "plone.org": 7, + "redhat.com": 7, + "oracle.com": 7, + "anonsvn.wireshark.org": 7, + "mail-archives.apache.org": 7, + "xenbits.xen.org": 6, + "postgresql.org": 6, + "twitter.com": 6, + "git.openssl.org": 6, + "jvn.jp": 6, + "rubygems.org": 6, + "symfony.com": 6, + "www-01.ibm.com": 6, + "src.chromium.org": 6, + "tenable.com": 6, + "lists.apple.com": 6, + "packagist.org": 6, + "issues.redhat.com": 6, + "helpx.adobe.com": 6, + "phabricator.wikimedia.org": 6, + "review.openstack.org": 6, + "chromereleases.googleblog.com": 6, + "contao.org": 6, + "discuss.elastic.co": 6, + "security.openstack.org": 6, + "lkml.org": 6, + "medium.com": 6, + "nozominetworks.com": 6, + "grafana.com": 6, + "oval.cisecurity.org": 6, + "review.opendev.org": 6, + "advisory.checkmarx.net": 6, + "vupen.com": 6, + "launchpad.net": 6, + "openssl.org": 6, + "framework.zend.com": 6, + "samba.org": 6, + "talosintelligence.com": 6, + "zerodayinitiative.com": 6, + "bugs.eclipse.org": 6, + "gitlab.gnome.org": 6, + "nifi.apache.org": 6, + "x-stream.github.io": 6, + "tanzu.vmware.com": 6, + "jvndb.jvn.jp": 6, + "opendev.org": 6, + "bugzilla.kernel.org": 6, + "sourceware.org": 6, + "codereview.chromium.org": 6, + "research.jfrog.com": 6, + "discuss.rubyonrails.org": 6, + "spring.io": 6, + "twcert.org.tw": 6, + "googlechromereleases.blogspot.com": 6, + "sec.cloudapps.cisco.com": 6, + "issues.jboss.org": 6, + "simplesamlphp.org": 6, + "mail-archive.com": 6, + "docs.saltstack.com": 6, + "docs.shopware.com": 6, + "kb.isc.org": 6, + "support.f5.com": 6, + "kernel.org": 6, + "osvdb.org": 6, + "cwiki.apache.org": 6, + "hg.moinmo.in": 6, + "fluidattacks.com": 6, + "gitlab.eclipse.org": 6, + "nuget.org": 6, + "lists.openstack.org": 6, + "developer.mozilla.org": 6, + "youtube.com": 6, + "drive.google.com": 6, + "owasp.org": 6, + "vaadin.com": 6, + "mercurial-scm.org": 6, + "mandriva.com": 6, + "archives.neohapsis.com": 6, + "hg.code.sf.net": 6, + "curl.haxx.se": 6, + "pillow.readthedocs.io": 6, + "struts.apache.org": 6, + "vulncheck.com": 6, + "webkitgtk.org": 6, + "documentation.concretecms.org": 6, + "patchstack.com": 6, + "patchwork.kernel.org": 6, + "hg.graphicsmagick.org": 6, + "source.android.com": 6, + "liferay.dev": 6, + "nodesecurity.io": 6, + "web.nvd.nist.gov": 6, + "jira.mongodb.org": 6, + "cve.org": 6, + "core.trac.wordpress.org": 6, + "news.ycombinator.com": 6, + "downloads.asterisk.org": 6, + "tomcat.apache.org": 6, + "kb.cert.org": 6, + "cert-portal.siemens.com": 6, + "devhub.checkmarx.com": 6, + "lists.nongnu.org": 6, + "squid-cache.org": 6, + "ink-desk-28f.notion.site": 6, + "h20566.www2.hpe.com": 6, + "intel.com": 6, + "docs.djangoproject.com": 6, + "go.googlesource.com": 6, + "herolab.usd.de": 6, + "matrix.org": 6, + "talosintel.com": 6, + "vapid.dhs.org": 6, + "ckeditor.com": 6, + "cwe.mitre.org": 6, + "pagure.io": 6, + "cisa.gov": 6, + "weblog.rubyonrails.org": 6, + "markmail.org": 6, + "issues.liferay.com": 6, + "jspwiki-wiki.apache.org": 6, + "gitbox.apache.org": 6, + "bugs.gentoo.org": 5, + "docs.microsoft.com": 5, + "puppet.com": 5, + "incibe.es": 5, + "cxf.apache.org": 5, + "bugzilla.wikimedia.org": 5, + "mvnrepository.com": 5, + "usom.gov.tr": 5, + "bz.apache.org": 5, + "nodejs.org": 5, + "syzkaller.appspot.com": 5, + "vuln.ryotak.me": 5, + "docs.google.com": 5, + "spinics.net": 5, + "auth0.com": 5, + "basercms.net": 5, + "next-auth.js.org": 5, + "plugins.jenkins.io": 5, + "libreoffice.org": 5, + "tiny.cloud": 5, + "hg.openjdk.java.net": 5, + "owncloud.org": 5, + "community.otrs.com": 5, + "git.openstack.org": 5, + "osgeo-org.atlassian.net": 5, + "bugzilla.gnome.org": 5, + "forum.xpdfreader.com": 5, + "istio.io": 5, + "developers.ibexa.co": 5, + "portswigger.net": 5, + "wpvulndb.com": 5, + "issues.chromium.org": 5, + "advisories.mageia.org": 5, + "doc.powerdns.com": 5, + "patchwork.ozlabs.org": 5, + "ec-cube.net": 5, + "phpbb.com": 5, + "rfc-editor.org": 5, + "synology.com": 5, + "ocert.org": 5, + "mail-archives.us.apache.org": 5, + "crates.io": 5, + "en.wikipedia.org": 5, + "issues.asterisk.org": 5, + "jira.spring.io": 5, + "secuniaresearch.flexerasoftware.com": 5, + "tools.ietf.org": 5, + "wiki.jenkins-ci.org": 5, + "youtu.be": 5, + "camel.apache.org": 5, + "0dd.zone": 5, + "blog.gitea.io": 5, + "bugs.ghostscript.com": 5, + "bugzilla.novell.com": 5, + "developer.joomla.org": 5, + "gitlab.torproject.org": 5, + "lists.wikimedia.org": 5, + "trac.torproject.org": 5, + "wpscan.com": 5, + "ibm.com": 5, + "trac.roundcube.net": 5, + "pidgin.im": 5, + "git.ghostscript.com": 5, + "gerrit.wikimedia.org": 5, + "gitlab.freedesktop.org": 5, + "hiddenlayer.com": 5, + "support.hpe.com": 5, + "thread.gmane.org": 5, + "cvcn.gov.it": 5, + "activemq.apache.org": 5, + "article.gmane.org": 5, + "bakery.cakephp.org": 5, + "bugs.python.org": 5, + "chromium.googlesource.com": 5, + "docs.github.com": 5, + "docs.rs": 5, + "grimthereaperteam.medium.com": 5, + "mailman.nginx.org": 5, + "psirt.global.sonicwall.com": 5, + "selenic.com": 5, + "esecforte.com": 5, + "mend.io": 5, + "netsparker.com": 5, + "playframework.com": 5, + "securityreason.com": 5, + "files.opcfoundation.org": 5, + "lgtm.com": 5, + "stackoverflow.com": 5, + "tools.cisco.com": 5, + "redmine.org": 5, + "vicarius.io": 5, + "w1.fi": 5, + "blog.phusion.nl": 5, + "codeigniter4.github.io": 5, + "csirt.divd.nl": 5, + "documentation.centreon.com": 5, + "eprint.iacr.org": 5, + "git.gnome.org": 5, + "joel-malwarebenchmark.github.io": 5, + "magento.com": 5, + "mantisbt.org": 5, + "research.loginsoft.com": 5, + "neos.io": 5, + "syss.de": 5, + "xforce.iss.net": 5, + "advisories.nats.io": 5, + "bugs.tryton.org": 5, + "lists.launchpad.net": 5, + "salsa.debian.org": 5, + "saltproject.io": 5, + "shopware.com": 5, + "shawroot.cc": 5, + "blog.ripstech.com": 5, + "blog.sonarsource.com": 5, + "cheatsheetseries.owasp.org": 5, + "docs.craftercms.org": 5, + "docs.umbraco.com": 5, + "go-review.googlesource.com": 5, + "kc.mcafee.com": 5, + "labs.integrity.pt": 5, + "pyup.io": 5, + "rubysec.com": 5, + "support.ntp.org": 5, + "wiki.openstack.org": 5, + "concretecms.org": 5, + "darkmatter.ae": 5, + "linkedin.com": 5, + "otrs.com": 5, + "privoxy.org": 5, + "sonarsource.com": 5, + "trustwave.com": 5, + "vmware.com": 5, + "blog.rubygems.org": 5, + "karmainsecurity.com": 5, + "cloud.google.com": 5, + "docs.python.org": 5, + "git-wip-us.apache.org": 5, + "nvidia.custhelp.com": 5, + "pastebin.com": 5, + "pypi.python.org": 5, + "python-security.readthedocs.io": 5, + "rumbling-slice-eb0.notion.site": 5, + "rapid7.com": 5, + "advisory.dw1.io": 5, + "akka.io": 5, + "android.googlesource.com": 5, + "argo-cd.readthedocs.io": 5, + "blog.fuzzing-project.org": 5, + "blog.laravel.com": 5, + "codeberg.org": 5, + "docs.cilium.io": 5, + "docs.docker.com": 5, + "docs.wagtail.org": 5, + "downloads.powerdns.com": 5, + "electronjs.org": 5, + "forums.rancher.com": 5, + "irssi.org": 5, + "kallithea-scm.org": 5, + "legalhackers.com": 5, + "lists.freedesktop.org": 5, + "mellium.im": 5, + "rubyonrails.org": 5, + "spark.apache.org": 5, + "storyboard.openstack.org": 5, + "subversion.apache.org": 5, + "tls.mbed.org": 5, + "sunsolve.sun.com": 5, + "wiki.ubuntu.com": 5, + "apollographql.com": 5, + "herodevs.com": 5, + "imagemagick.org": 5, + "kde.org": 5, + "qualys.com": 5, + "usenix.org": 5, + "wizlynxgroup.com": 5, + "x41-dsec.de": 5, + "gentoo.org": 5, + "git.videolan.org": 5, + "moinmo.in": 5, + "puppetlabs.com": 5, + "arxiv.org": 5, + "blog.getbootstrap.com": 5, + "review.gluster.org": 5, + "roy.marples.name": 5, + "stackblitz.com": 5, + "facebook.com": 5, + "git.savannah.gnu.org": 5, + "blog.torproject.org": 5, + "codepen.io": 5, + "codeql.github.com": 5, + "commons.apache.org": 5, + "confluence.atlassian.com": 5, + "cs.opensource.google": 5, + "doc.akka.io": 5, + "docs.opennms.com": 5, + "docs.openstack.org": 5, + "docs.silverstripe.org": 5, + "doc.traefik.io": 5, + "doyensec.com": 5, + "foss.heptapod.net": 5, + "gcc.gnu.org": 5, + "git.sheetjs.com": 5, + "hackmd.io": 5, + "ics-cert.kaspersky.com": 5, + "kiwitcms.org": 5, + "kjur.github.io": 5, + "lists.gnupg.org": 5, + "logback.qos.ch": 5, + "logging.apache.org": 5, + "mail.python.org": 5, + "metacpan.org": 5, + "neo4j.com": 5, + "opcfoundation.org": 5, + "ostorlab.co": 5, + "palletsprojects.com": 5, + "portal.liferay.dev": 5, + "research.securitum.com": 5, + "sites.google.com": 5, + "source.codeaurora.org": 5, + "support.sonatype.com": 5, + "twistedmatrix.com": 5, + "edoardoottavianelli.it": 5, + "sec-consult.com": 5, + "sudo.ws": 5, + "vulnerability-lab.com": 5, + "yiiframework.com": 5, + "www-1.ibm.com": 5, + "cherrypy.org": 5, + "vapidlabs.com": 5, + "archiva.apache.org": 5, + "bugs.cacti.net": 5, + "docs.moodle.org": 5, + "freeradius.org": 5, + "git.php.net": 5, + "hmarco.org": 5, + "hyp3rlinx.altervista.org": 5, + "lists.bestpractical.com": 5, + "php.net": 5, + "qpid.apache.org": 5, + "blog.clamav.net": 5, + "codex.wordpress.org": 5, + "cxsecurity.com": 5, + "security.libvirt.org": 5, + "extensions.typo3.org": 5, + "gnunet.org": 5, + "pretix.eu": 5, + "software.intel.com": 5, + "syncope.apache.org": 5, + "apache.org": 5, + "blackhat.com": 5, + "bleepingcomputer.com": 5, + "cyrusimap.org": 5, + "openssh.com": 5, + "arubanetworks.com": 5, + "code610.blogspot.com": 5, + "code.djangoproject.com": 5, + "karaf.apache.org": 5, + "lists.openwall.net": 5, + "netty.io": 5, + "packetstormsecurity.org": 5, + "arstechnica.com": 5, + "aws.amazon.com": 5, + "benjamin-bouchet.com": 5, + "blog.jquery.com": 5, + "bugs.jquery.com": 5, + "bugs.php.net": 5, + "bugs.torproject.org": 5, + "cert.pl": 5, + "community.grafana.com": 5, + "community.shopware.com": 5, + "cves.at": 5, + "datatracker.ietf.org": 5, + "docs.geoserver.org": 5, + "docs.sentry.io": 5, + "edugit.org": 5, + "fedorahosted.org": 5, + "fetch.spec.whatwg.org": 5, + "froala.com": 5, + "gerrit.libreoffice.org": 5, + "git1-us-west.apache.org": 5, + "grimhacker.com": 5, + "hg.reportlab.com": 5, + "kb.pulsesecure.net": 5, + "laravel.com": 5, + "lwn.net": 5, + "mail.openvswitch.org": 5, + "matrix-org.github.io": 5, + "nukeviet.vn": 5, + "portal.microfocus.com": 5, + "research.nccgroup.com": 5, + "rt.cpan.org": 5, + "s.apache.org": 5, + "search.maven.org": 5, + "security.docs.wso2.com": 5, + "share.ez.no": 5, + "shibboleth.net": 5, + "solr.apache.org": 5, + "starlabs.sg": 5, + "support.microsoft.com": 5, + "support.zabbix.com": 5, + "sysdream.com": 5, + "tech.feedyourhead.at": 5, + "dolibarr.org": 5, + "hashicorp.com": 5, + "opennms.com": 5, + "passbolt.com": 5, + "reddit.com": 5, + "sqlite.org": 5, + "strongswan.org": 5, + "hawktesters.com": 5, + "topquadrant.com": 5, + "blog.mindedsecurity.com": 5, + "dev.eclipse.org": 5, + "downloads.digium.com": 5, + "permalink.gmane.org": 5, + "0xacab.org": 5, + "about.gitlab.com": 5, + "alephsecurity.com": 5, + "blog.cloudflare.com": 5, + "bugs.freedesktop.org": 5, + "bugs.kde.org": 5, + "community.openvpn.net": 5, + "core.spip.net": 5, + "docs.oracle.com": 5, + "dumpco.re": 5, + "edk2-docs.gitbooks.io": 5, + "forum.cosmos.network": 5, + "git.ffmpeg.org": 5, + "golang.org": 5, + "hex.pm": 5, + "lists.x.org": 5, + "me.sap.com": 5, + "pdfium.googlesource.com": 5, + "peps.python.org": 5, + "portal.perforce.com": 5, + "roundcube.net": 5, + "trac.ffmpeg.org": 5, + "trac.pjsip.org": 5, + "svn.cacti.net": 5, + "couchbase.com": 5, + "nlnetlabs.nl": 5, + "prevanders.net": 5, + "sourceclear.com": 5, + "suse.com": 5, + "novell.com": 5, + "openafs.org": 5, + "tryton.org": 5, + "zope.org": 5, + "blog.plataformatec.com.br": 5, + "blog.trendmicro.com": 5, + "geronimo.apache.org": 5, + "hg.tryton.org": 5, + "issues.roundup-tracker.org": 5, + "akerva.com": 5, + "api.slack.com": 5, + "backdropcms.org": 5, + "blog.certimetergroup.com": 5, + "blog.jetbrains.com": 5, + "codereview.qt-project.org": 5, + "codewhitesec.blogspot.com": 5, + "community.librenms.org": 5, + "craftcms.com": 5, + "discord.com": 5, + "discuss.flarum.org": 5, + "discuss.tryton.org": 5, + "dl.acm.org": 5, + "docs.aws.amazon.com": 5, + "docs.pinot.apache.org": 5, + "docs.projectdiscovery.io": 5, + "documentation.bonitasoft.com": 5, + "duo.com": 5, + "elixirforum.com": 5, + "extensions.xwiki.org": 5, + "ezplatform.com": 5, + "git.launchpad.net": 5, + "gitlord.com": 5, + "git.shibboleth.net": 5, + "gnutls.org": 5, + "gxx777.github.io": 5, + "ibb.co": 5, + "i.ibb.co": 5, + "internetcomputer.org": 5, + "issues.opennms.org": 5, + "jira.qos.ch": 5, + "jsoup.org": 5, + "kiali.io": 5, + "lab.louiz.org": 5, + "labs.bishopfox.com": 5, + "linkis.apache.org": 5, + "mattermost.com": 5, + "mycvee.blogspot.com": 5, + "notes.sjtu.edu.cn": 5, + "opcfoundation-onlineapplications.org": 5, + "openid.net": 5, + "patch-diff.githubusercontent.com": 5, + "pho3n1x-web.github.io": 5, + "podalirius.net": 5, + "portals.apache.org": 5, + "projects.eclipse.org": 5, + "pulsesecurity.co.nz": 5, + "quarkus.io": 5, + "reactjs.org": 5, + "review.typo3.org": 5, + "salvatoresecurity.com": 5, + "securelayer7.net": 5, + "secure.ucc.asn.au": 5, + "shrouded-trowel-50c.notion.site": 5, + "ssd-disclosure.com": 5, + "strapi.io": 5, + "surrealdb.com": 5, + "tailscale.com": 5, + "tufangungor.github.io": 5, + "support.avaya.com": 5, + "webargs.readthedocs.io": 5, + "whitehub.net": 5, + "ait.ac.at": 5, + "checkmarx.com": 5, + "cloudbees.com": 5, + "cloudera.com": 5, + "codeigniter.com": 5, + "digital.security": 5, + "dulwich.io": 5, + "freeipa.org": 5, + "gerritcodereview.com": 5, + "ghostccamm.com": 5, + "horizon3.ai": 5, + "imperva.com": 5, + "mediawiki.org": 5, + "openmicroscopy.org": 5, + "schedmd.com": 5, + "wwws.nightwatchcybersecurity.com": 5, + "synacktiv.com": 5, + "synopsys.com": 5, + "tigera.io": 5, + "twistlock.com": 5, + "w3.org": 5, + "verneet.com": 5, + "dnnsoftware.com": 5, + "halfdog.net": 5, + "hitachi.co.jp": 5, + "reviewboard.org": 5, + "us-cert.gov": 5, + "depot.galaxyproject.org": 5, + "codecanyon.net": 5, + "marketplace.visualstudio.com": 5, + "docs.telerik.com": 5, + "support.hcltechsw.com": 5, + "watchguard.com": 5, + "bazaar.launchpad.net": 5, + "comments.gmane.org": 5, + "db.apache.org": 5, + "debbugs.gnu.org": 5, + "emberjs.com": 5, + "git.ganeti.org": 5, + "lists.thekelleys.org.uk": 5, + "phpmyadmin.git.sourceforge.net": 5, + "projects.puppetlabs.com": 5, + "anonscm.debian.org": 5, + "apereo.github.io": 5, + "blogs.apache.org": 5, + "bugzilla.opensuse.org": 5, + "build.prestashop.com": 5, + "community.rapid7.com": 5, + "cryptography.io": 5, + "cybersecurityworks.com": 5, + "dev.mysql.com": 5, + "docs.opencast.org": 5, + "docs.pagure.org": 5, + "docs.powerdns.com": 5, + "ftp.openbsd.org": 5, + "gitlab.labs.nic.cz": 5, + "git.linuxtv.org": 5, + "git.spip.net": 5, + "googleprojectzero.blogspot.com": 5, + "helm.sh": 5, + "hermes.opensuse.org": 5, + "hg.prosody.im": 5, + "isc.sans.edu": 5, + "issues.jenkins-ci.org": 5, + "jfrog.com": 5, + "jira.hdfgroup.org": 5, + "jira.springsource.org": 5, + "lists.01.org": 5, + "lists.lysator.liu.se": 5, + "lists.torproject.org": 5, + "nealpoole.com": 5, + "nokogiri.org": 5, + "novysodope.github.io": 5, + "somevulnsofadlab.blogspot.jp": 5, + "source.ikiwiki.branchable.com": 5, + "patchwork.linuxtv.org": 5, + "prosody.im": 5, + "pub.freerdp.com": 5, + "research.checkpoint.com": 5, + "rt-solutions.de": 5, + "tracker.debian.org": 5, + "user-images.githubusercontent.com": 5, + "ambionics.io": 5, + "bouncycastle.org": 5, + "freebsd.org": 5, + "mautic.org": 5, + "theregister.com": 5, + "trac.edgewall.org": 5, + "fujitsu.com": 5, + "gopivotal.com": 5, + "huawei.com": 5, + "ush.it": 5, + "wiki.shikangsi.com": 5, + "xenbits.xenproject.org": 5, + "bytium.com": 5, + "enpass.io": 5, + "archives.seul.org": 5, + "blog.npmjs.org": 5, + "blog.talosintel.com": 5, + "bugs.jqueryui.com": 5, + "dev.rubyonrails.org": 5, + "download.lighttpd.net": 5, + "freeimage.cvs.sourceforge.net": 5, + "googlechromereleases.blogspot.ca": 5, + "ijbswa.cvs.sourceforge.net": 5, + "issues.umbraco.org": 5, + "jira.codehaus.org": 5, + "jruby.org": 5, + "lcamtuf.coredump.cx": 5, + "lists.gnutls.org": 5, + "lists.qt-project.org": 5, + "lists.xen.org": 5, + "mx.gw.com": 5, + "octobercms.com": 5, + "openmeetings.apache.org": 5, + "pear.php.net": 5, + "projects.theforeman.org": 5, + "research.cs.wisc.edu": 5, + "airflow.apache.org": 5, + "santuario.apache.org": 5, + "appcheck-ng.com": 5, + "app.safebase.io": 5, + "app.snyk.io": 5, + "argoproj.github.io": 5, + "artifacthub.io": 5, + "authjs.dev": 5, + "savannah.nongnu.org": 5, + "bbpress.org": 5, + "belong2yourself.github.io": 5, + "blocksecteam.medium.com": 5, + "blog.cal1.cn": 5, + "blog.detectify.com": 5, + "blog.doyensec.com": 5, + "blog.gitea.com": 5, + "blog.jcoglan.com": 5, + "blog.jitendrapatro.me": 5, + "blog.jqueryui.com": 5, + "blog.jupyter.org": 5, + "blog.moku.fr": 5, + "blog.payara.fish": 5, + "blogs.securiteam.com": 5, + "blog.trailofbits.com": 5, + "breakingthe3ma.app": 5, + "brooklyn.apache.org": 5, + "buddypress.org": 5, + "bugs.oxid-esales.com": 5, + "bz.mercurial-scm.org": 5, + "cgit.kde.org": 5, + "channels.readthedocs.io": 5, + "chromium-review.googlesource.com": 5, + "ckan.org": 5, + "clojure.atlassian.net": 5, + "clusterfuzz-external.appspot.com": 5, + "community.snowflake.com": 5, + "community.sonarsource.com": 5, + "community.synopsys.com": 5, + "cupc4k3.medium.com": 5, + "developer.arm.com": 5, + "dev.liferay.com": 5, + "discuss.lightbend.com": 5, + "docs.getdbt.com": 5, + "docs.gradle.com": 5, + "docs.jboss.org": 5, + "docs.magnolia-cms.com": 5, + "docs.mattermost.com": 5, + "docs.opendaylight.org": 5, + "docs.rundeck.com": 5, + "docs.sixlabors.com": 5, + "docs.strapi.io": 5, + "docs.typo3.org": 5, + "docs.wso2.com": 5, + "duartecsantos.github.io": 5, + "eclipse.dev": 5, + "eslint.org": 5, + "excellium-services.com": 5, + "exment.net": 5, + "fatihhcelik.blogspot.com": 5, + "fortiguard.com": 5, + "forum.ghost.org": 5, + "forum.mautic.org": 5, + "gerrit.onosproject.org": 5, + "geth.ethereum.org": 5, + "ghost.org": 5, + "gist.githubusercontent.com": 5, + "git.eclipse.org": 5, + "git.haproxy.org": 5, + "github.dev": 5, + "github.openssl.org": 5, + "gitlab.kitware.com": 5, + "gitlab.ow2.org": 5, + "gitlab.redox-os.org": 5, + "git.opendaylight.org": 5, + "git-scm.com": 5, + "gitweb.gentoo.org": 5, + "gitweb.torproject.org": 5, + "glassfish.org": 5, + "go-vela.github.io": 5, + "grails.org": 5, + "gstreamer.freedesktop.org": 5, + "guidovranken.wordpress.com": 5, + "hadoop.apache.org": 5, + "hexdocs.pm": 5, + "hg.python.org": 5, + "htcondor-wiki.cs.wisc.edu": 5, + "ihacktoprotect.com": 5, + "ikiwiki.info": 5, + "infosecwriteups.com": 5, + "inspector.pypi.io": 5, + "invdos.net": 5, + "iotaa.cn": 5, + "ipsilon-project.org": 5, + "issues.shibboleth.net": 5, + "james.apache.org": 5, + "javaee.github.io": 5, + "jdbc.postgresql.org": 5, + "jekyllrb.com": 5, + "jena.apache.org": 5, + "jira.duraspace.org": 5, + "jira.hyperledger.org": 5, + "johnjhacking.com": 5, + "jolokia.org": 5, + "jupyterhub.readthedocs.io": 5, + "knightlab.northwestern.edu": 5, + "know.bishopfox.com": 5, + "kramdown.gettalong.org": 5, + "kubernetes.io": 5, + "kyverno.io": 5, + "laravel-news.com": 5, + "launchpadlibrarian.net": 5, + "learn.microsoft.com": 5, + "lf-opendaylight.atlassian.net": 5, + "lists.exim.org": 5, + "lists.quagga.net": 5, + "lists.samba.org": 5, + "lists.schedmd.com": 5, + "loopback.io": 5, + "mail.zope.org": 5, + "makandracards.com": 5, + "mariadb.com": 5, + "mosquitto.org": 5, + "mp.weixin.qq.com": 5, + "openfga.dev": 5, + "oxidforge.org": 5, + "pentest.com.tr": 5, + "phab.bots.miraheze.wiki": 5, + "plugins.craftcms.com": 5, + "polarssl.org": 5, + "spreecommerce.com": 5, + "public-inbox.org": 5, + "rancher.com": 5, + "redmine.openinfosecfoundation.org": 5, + "repo.mercurial-scm.org": 5, + "rhinosecuritylabs.com": 5, + "runkit.com": 5, + "sansec.io": 5, + "savannah.gnu.org": 5, + "seanmonstar.com": 5, + "security.FreeBSD.org": 5, + "seemann.io": 5, + "sl1nki.page": 5, + "sources.debian.org": 5, + "stackdiary.com": 5, + "support.hazelcast.com": 5, + "supportportal.juniper.net": 5, + "support.servicenow.com": 5, + "swarm.ptsecurity.com": 5, + "tauri.app": 5, + "thecybergeek.co.uk": 5, + "tib36.github.io": 5, + "storm.apache.org": 5, + "trovent.github.io": 5, + "trovent.io": 5, + "uima.apache.org": 5, + "unit42.paloaltonetworks.com": 5, + "unparalleled.eu": 5, + "support.blackberry.com": 5, + "uwsgi-docs.readthedocs.io": 5, + "vufind.org": 5, + "vulmon.com": 5, + "vuln.go.dev": 5, + "vuln.ryotak.net": 5, + "websecnerd.blogspot.in": 5, + "wha13.github.io": 5, + "whitehatck01.blogspot.com": 5, + "wiki.mageia.org": 5, + "wiki.phpbb.com": 5, + "wp-crontrol.com": 5, + "ag-grid.com": 5, + "apiman.io": 5, + "bentley.com": 5, + "bitvise.com": 5, + "bluetooth.com": 5, + "bookstackapp.com": 5, + "cnblogs.com": 5, + "cnvd.org.cn": 5, + "code-intelligence.com": 5, + "compass-security.com": 5, + "django-cms.org": 5, + "doctrine-project.org": 5, + "eclipse.org": 5, + "elastic.co": 5, + "envoyproxy.io": 5, + "exploitalert.com": 5, + "fastly.com": 5, + "fzi.de": 5, + "getastra.com": 5, + "graylog.org": 5, + "htbridge.com": 5, + "kingkk.com": 5, + "libssh.org": 5, + "nccgroup.trust": 5, + "nginx.com": 5, + "openpolicyagent.org": 5, + "pac4j.org": 5, + "pgadmin.org": 5, + "phpmyfaq.de": 5, + "roundup-tracker.org": 5, + "rubydoc.info": 5, + "saltstack.com": 5, + "secpod.com": 5, + "shielder.it": 5, + "smarty.net": 5, + "smilecdr.com": 5, + "soluble.ai": 5, + "star123.top": 5, + "sumor.top": 5, + "swascan.com": 5, + "teeworlds.com": 5, + "verot.net": 5, + "videolan.org": 5, + "xwiki.org": 5, + "yuque.com": 5, + "yarnpkg.com": 5, + "zeppelin.apache.org": 5, + "unomi.apache.org": 5, + "web2py.com": 5, + "web.mit.edu": 5, + "chiark.greenend.org.uk": 5, + "conostix.com": 5, + "daimacn.com": 5, + "dojotoolkit.org": 5, + "igniterealtime.org": 5, + "logilab.org": 5, + "swi-prolog.org": 5, + "yuiblog.com": 5, + "xerces.apache.org": 5, + "yuilibrary.com": 5, + "zaranshaikh.blogspot.com": 5, + "checkmk.com": 5, + "support.eset.com": 5, + "git.canopsis.net": 5, + "forescout.com": 5, + "mitel.com": 5, + "cert-in.org.in": 5, + "schneider-elektronik.de": 5, + "security-advisory.acronis.com": 5, + "activemq.2283324.n4.nabble.com": 5, + "apache-ignite-developers.2346864.n4.nabble.com": 5, + "baraktawily.blogspot.co.il": 5, + "blog.diniscruz.com": 5, + "blog.kotowicz.net": 5, + "blog.neargle.com": 5, + "blog.nodejs.org": 5, + "blog.talosintelligence.com": 5, + "borgbackup.readthedocs.io": 5, + "bugs.proftpd.org": 5, + "cgit.drupalcode.org": 5, + "erlang.org": 5, + "extplorer.net": 5, + "foxglovesecurity.com": 5, + "freshmeat.net": 5, + "googlechromereleases.blogspot.de": 5, + "h20000.www2.hp.com": 5, + "htmlpurifier.org": 5, + "itrc.hp.com": 5, + "kalilinux.co": 5, + "lists.horde.org": 5, + "lists.suse.com": 5, + "lists.ucc.gu.uwa.edu.au": 5, + "lkml.iu.edu": 5, + "mercurial.selenic.com": 5, + "miki.it": 5, + "openocd.zylin.com": 5, + "phpmyadmin.svn.sourceforge.net": 5, + "poi.apache.org": 5, + "redmine.lighttpd.net": 5, + "repo.or.cz": 5, + "rubyforge.org": 5, + "aka.ms": 5, + "sakurity.com": 5, + "blog.filippo.io": 5, + "blog.ircmaxell.com": 5, + "blog.justinbull.ca": 5, + "blogs.akamai.com": 5, + "blog.sonatype.com": 5, + "blog.spip.net": 5, + "blog.srcclr.com": 5, + "brakemanscanner.org": 5, + "bridge.grumpy-troll.org": 5, + "buer.haus": 5, + "bugs.dojotoolkit.org": 5, + "bugs.openjdk.java.net": 5, + "bugs.otr.im": 5, + "bugzilla.proxmox.com": 5, + "collectiveidea.com": 5, + "connect2id.com": 5, + "coredns.io": 5, + "cosmosofcyberspace.github.io": 5, + "delvingbitcoin.org": 5, + "devco.re": 5, + "dev.icinga.org": 5, + "docs.cometbft.com": 5, + "docs.couchbase.com": 5, + "docs.getindico.io": 5, + "dovecot.org": 5, + "download.samba.org": 5, + "forge.glpi-project.org": 5, + "ftp.suse.com": 5, + "git.blender.org": 5, + "git.enlightenment.org": 5, + "git.musl-libc.org": 5, + "git.samba.org": 5, + "git.zx2c4.com": 5, + "grsecurity.net": 5, + "hackdefense.com": 5, + "hg.mozilla.org": 5, + "hibernate.atlassian.net": 5, + "ics-cert.us-cert.gov": 5, + "int21.de": 5, + "issues.igniterealtime.org": 5, + "issues.rpath.com": 5, + "kb.juniper.net": 5, + "kyberslash.cr.yp.to": 5, + "labs.twistedmatrix.com": 5, + "ledgersmb.org": 5, + "lib.openmpt.org": 5, + "lists.dns-oarc.net": 5, + "lists.fedorahosted.org": 5, + "lists.linuxfoundation.org": 5, + "lists.xapian.org": 5, + "minerva.crocs.fi.muni.cz": 5, + "mta.openssl.org": 5, + "somevulnsofadlab.blogspot.com": 5, + "oss-fuzz.com": 5, + "oss.oracle.com": 5, + "source.git-annex.branchable.com": 5, + "pub.dev": 5, + "quickview.cloudapps.cisco.com": 5, + "robotattack.org": 5, + "security.googleblog.com": 5, + "shells.systems": 5, + "sintonen.fi": 5, + "skia.googlesource.com": 5, + "staaldraad.github.io": 5, + "store.shopware.com": 5, + "sumofpwn.nl": 5, + "support.lenovo.com": 5, + "trac.osgeo.org": 5, + "trac.webkit.org": 5, + "trac.xiph.org": 5, + "udiniya.wordpress.com": 5, + "support.novell.com": 5, + "weechat.org": 5, + "withatwist.dev": 5, + "codeaurora.org": 5, + "exim.org": 5, + "foxmole.com": 5, + "genivia.com": 5, + "gosecure.net": 5, + "home-assistant.io": 5, + "quagga.net": 5, + "tarlogic.com": 5, + "varnish-cache.org": 5, + "xorl.wordpress.com": 5, + "zenar.io": 5, + "zone.spip.net": 5, + "zyan.scripts.mit.edu": 5, + "theforeman.org": 5, + "tracker.firebirdsql.org": 5, + "trubka.network.cz": 5, + "wiki.mahara.org": 5, + "cacti.net": 5, + "eterna.com.au": 5, + "eweek.com": 5, + "ghostscript.com": 5, + "h5l.org": 5, + "ietf.org": 5, + "information-security.fr": 5, + "iwantacve.cn": 5, + "links.org": 5, + "mutt.org": 5, + "slackware.com": 5, + "thewildbeast.co.uk": 5, + "tornadoweb.org": 5, + "znuny.com": 5, + "apps.apple.com": 5, + "dontvacuum.me": 5, + "wiz.io": 5, + "tsc-soft.co.jp": 5, + "2016.hack.lu": 5, + "agrrrdog.blogspot.com": 5, + "alexcrack.com": 5, + "apache-spark-developers-list.1001551.n3.nabble.com": 5, + "appwrite.com": 5, + "archive.hack.lu": 5, + "argo.com": 5, + "bandoche.com": 5, + "bilishim.com": 5, + "blackboxexporter.com": 5, + "blog.amossys.fr": 5, + "blog.angularjs.org": 5, + "blog.apps.npr.org": 5, + "blog.bestpractical.com": 5, + "blog.checkpoint.com": 5, + "blog.csdn.net": 5, + "blog.datomic.com": 5, + "blog.emaze.net": 5, + "blog.intothesymmetry.com": 5, + "blog.portswigger.net": 5, + "blog.recurity-labs.com": 5, + "blog.securelayer7.net": 5, + "boltcms.com": 5, + "breaktoprotect.blogspot.com": 5, + "bugs.jython.org": 5, + "bugs.ledger-cli.org": 5, + "buildbot.net": 5, + "bzr.linuxfoundation.org": 5, + "ca17.com": 5, + "census-labs.com": 5, + "centreon.com": 5, + "ceriksen.com": 5, + "chargen.matasano.com": 5, + "chrony.tuxfamily.org": 5, + "code.call-cc.org": 5, + "community.ca.com": 5, + "community.impresscms.org": 5, + "cpansearch.perl.org": 5, + "craft.com": 5, + "crafter.com": 5, + "cvs.moodle.org": 5, + "cvsweb.netbsd.org": 5, + "cyberworldmirror.com": 5, + "davidsopaslabs.blogspot.com": 5, + "dev.deluge-torrent.org": 5, + "dfn.dl.sourceforge.net": 5, + "directory.apache.org": 5, + "doc.scrapy.org": 5, + "doc.silverstripe.org": 5, + "docs.info.apple.com": 5, + "docs.python-requests.org": 5, + "docs.withknown.com": 5, + "dolibarr.com": 5, + "dotclear.org": 5, + "dx.doi.org": 5, + "elixir.ematia.de": 5, + "embed.plnkr.co": 5, + "en.0day.today": 5, + "encode.com": 5, + "etcd.com": 5, + "ethereum.com": 5, + "evilpacket.net": 5, + "exfiltrated.com": 5, + "extendedsubset.com": 5, + "fitnesse.org": 5, + "flyingmana.de": 5, + "freecode.com": 5, + "ghost.com": 5, + "gilacms.com": 5, + "git-blame.blogspot.com": 5, + "git.deluge-torrent.org": 5, + "githubcommherflower.com": 5, + "git.postgresql.org": 5, + "go-ethereum.com": 5, + "gogs.io": 5, + "gollum.com": 5, + "grailsblog.objectcomputing.com": 5, + "groovy-lang.org": 5, + "ha.cker.info": 5, + "haxx.ml": 5, + "heartex.com": 5, + "help.plot.ly": 5, + "hidden-one.co.in": 5, + "holisticinfosec.org": 5, + "hsqldb.org": 5, + "icecoder.com": 5, + "info.tiki.org": 5, + "jakarta.apache.org": 5, + "jeecg-boot.com": 5, + "jinja.pocoo.org": 5, + "joplin.com": 5, + "juddi.apache.org": 5, + "labelstud.io": 5, + "labs.mwrinfosecurity.com": 5, + "lambdaops.com": 5, + "langchain.com": 5, + "lavalite.com": 5, + "lcamtuf.blogspot.ca": 5, + "lesscss.org": 5, + "libvirt.org": 5, + "liferay.com": 5, + "linux.oracle.com": 5, + "lists.live555.com": 5, + "lists.mutt.org": 5, + "lists.roaringpenguin.com": 5, + "lists.typo3.org": 5, + "lua.2524044.n2.nabble.com": 5, + "lxml.de": 5, + "maven.apache.org": 5, + "mina.apache.org": 5, + "mumble.info": 5, + "n8.tumblr.com": 5, + "nevado.skyscreamer.org": 5, + "ngenuity-is.com": 5, + "nginx.org": 5, + "nodejs.com": 5, + "openapi-generator.com": 5, + "openbsd.org": 5, + "opennlp.apache.org": 5, + "ph0rse.me": 5, + "processwire.com": 5, + "prometheus.com": 5, + "pyload.com": 5, + "pypi.doubanio.com": 5, + "pypinksign.com": 5, + "python.6.x6.nabble.com": 5, + "qcubed.com": 5, + "rails.lighthouseapp.com": 5, + "raneto.com": 5, + "request-baskets.com": 5, + "resque.com": 5, + "roundup.svn.sourceforge.net": 5, + "ruffsecurity.blogspot.com": 5, + "0ang3el.blogspot.ru": 5, + "2018.zeronights.ru": 5, + "98587329.github.io": 5, + "aaltodoc.aalto.fi": 5, + "accumulo.apache.org": 5, + "acloudtree.com": 5, + "adamcaudill.com": 5, + "advisories.gitlab.com": 5, + "advisories.octopus.com": 5, + "aetsu.github.io": 5, + "agent-js.icp.xyz": 5, + "sailsjs.org": 5, + "aisec.today": 5, + "aleksis.org": 5, + "alesandroortiz.com": 5, + "alexsecurity.rocks": 5, + "alicangonullu.org": 5, + "anisiosantos.me": 5, + "anongit.mindrot.org": 5, + "ant.apache.org": 5, + "antmedia.io": 5, + "apidoc.gitee.com": 5, + "apidock.com": 5, + "apiiro.com": 5, + "app.intigriti.com": 5, + "appwrite.io": 5, + "atlaskit.atlassian.com": 5, + "attackerkb.com": 5, + "at-trustit.tuv.at": 5, + "autobahn.readthedocs.io": 5, + "avideo.tube": 5, + "backbonejs.org": 5, + "bad.code.blog": 5, + "badpackets.net": 5, + "bandit.readthedocs.io": 5, + "baomidou.com": 5, + "beanvalidation.org": 5, + "becomepentester.blogspot.ae": 5, + "bishopfox.com": 5, + "bitbucket.hdfgroup.org": 5, + "bittherapy.net": 5, + "blakeembrey.com": 5, + "blitiri.com.ar": 5, + "blog.0xzon.dev": 5, + "blog.appsecco.com": 5, + "blog.bentkowski.info": 5, + "blog.bssi.fr": 5, + "blog.caller.xyz": 5, + "blog.chebuya.com": 5, + "blog.compass-security.com": 5, + "blog.daniel-ruf.de": 5, + "blog.effectrenan.com": 5, + "blog.ethereum.org": 5, + "blog.exodusintel.com": 5, + "blog.gdssecurity.com": 5, + "blog.gradle.org": 5, + "blog.hackeriet.no": 5, + "blog.hackingforce.com.br": 5, + "blog.ipfs.io": 5, + "blog.isosceles.com": 5, + "blog.jiguang.xyz": 5, + "blog.librenms.org": 5, + "blog.litespeedtech.com": 5, + "blog.mevsec.com": 5, + "blog.nettitude.com": 5, + "blog.orange.tw": 5, + "blog.ostorlab.co": 5, + "blog.phpbb.com": 5, + "blog.pusher.com": 5, + "blog.qualys.com": 5, + "blog.sentry.io": 5, + "blogs.gnome.org": 5, + "blog.slonser.info": 5, + "blog.smithsecurity.biz": 5, + "blogs.opera.com": 5, + "blogs.sap.com": 5, + "blog.truesec.com": 5, + "blog.xss.am": 5, + "boats.gitlab.io": 5, + "boho.or.kr": 5, + "book.hacktricks.xyz": 5, + "brave.com": 5, + "breakandpray.com": 5, + "bsg.tech": 5, + "bto.bluecoat.com": 5, + "bugs.bitlbee.org": 5, + "bugs.libssh.org": 5, + "bugs.limesurvey.org": 5, + "bugs.otrs.org": 5, + "bugs.ruby-lang.org": 5, + "bugzilla.libav.org": 5, + "bundler.io": 5, + "burninatorsec.blogspot.com": 5, + "buttercup.pw": 5, + "caddy.community": 5, + "caddyserver.com": 5, + "scala-lang.org": 5, + "cardaci.xyz": 5, + "carl1l.github.io": 5, + "scarybeastsecurity.blogspot.de": 5, + "casdoor.org": 5, + "c-c-a.org": 5, + "cdn2.hubspot.net": 5, + "cdn.datatables.net": 5, + "cdn.sheetjs.com": 5, + "cert.enea.pl": 5, + "cgit.freebsd.org": 5, + "cgit.freedesktop.org": 5, + "chaos.social": 5, + "chartkick.com": 5, + "checkstyle.org": 5, + "chmod744.super.site": 5, + "chocapikk.com": 5, + "clerk.com": 5, + "clojars.org": 5, + "cloud-trustit.spp.at": 5, + "codeburst.io": 5, + "code.jeremyevans.net": 5, + "codex.bbpress.org": 5, + "codex.buddypress.org": 5, + "com0t.github.io": 5, + "commonmark.thephpleague.com": 5, + "community.contao.org": 5, + "community.developer.atlassian.com": 5, + "community.gravitee.io": 5, + "community.traefik.io": 5, + "community.veracode.com": 5, + "concretecms.com": 5, + "contrastsecurity.com": 5, + "cortexmetrics.io": 5, + "cowtowncoder.medium.com": 5, + "cratedb.com": 5, + "cryptosense.com": 5, + "csrc.nist.gov": 5, + "cube01.io": 5, + "cupc4k3.lol": 5, + "cure53.de": 5, + "cve.anastasi.link": 5, + "cve.naver.com": 5, + "cve.nstsec.com": 5, + "cyllective.com": 5, + "danielfett.de": 5, + "dannewitz.ninja": 5, + "darkbit.io": 5, + "datasette.io": 5, + "datnlq.gitbook.io": 5, + "deadsh0t.medium.com": 5, + "demo.ripstech.com": 5, + "deps.dev": 5, + "devdocs.magento.com": 5, + "devel0pment.de": 5, + "developer.apple.com": 5, + "developer.hashicorp.com": 5, + "developer.jboss.org": 5, + "developers.cloudflare.com": 5, + "developers.google.com": 5, + "developer.shopware.com": 5, + "developers.yubico.com": 5, + "developer.woocommerce.com": 5, + "develop.sentry.dev": 5, + "dev.gajim.org": 5, + "dev.to": 5, + "diff.coditsu.io": 5, + "diff.hex.pm": 5, + "digi.ninja": 5, + "directus.io": 5, + "discourse.vtk.org": 5, + "discuss.istio.io": 5, + "dist.apache.org": 5, + "dist.plone.org": 5, + "dnsdist.org": 5, + "doc.clickup.com": 5, + "doc.rust-lang.org": 5, + "docs.centreon.com": 5, + "docs.ceph.com": 5, + "docs.ckan.org": 5, + "docs.dapr.io": 5, + "docs.dask.org": 5, + "docs.dependencytrack.org": 5, + "docs.directus.io": 5, + "docs.flyte.org": 5, + "docs.gitlab.com": 5, + "docs.gofiber.io": 5, + "docs.gravityforms.com": 5, + "docs.libp2p.io": 5, + "docs.locust.io": 5, + "docs.mulesoft.com": 5, + "docs.nautobot.com": 5, + "docs.nginx.com": 5, + "docs.parseplatform.org": 5, + "docs.pylonsproject.org": 5, + "docs.r3.com": 5, + "docs.saltproject.io": 5, + "docs.scrapy.org": 5, + "docs.securesauce.dev": 5, + "docs.sigstore.dev": 5, + "docs.snowflake.com": 5, + "docs.spring.io": 5, + "docs.sqlalchemy.org": 5, + "docs.sympy.org": 5, + "docs.tigergraph.com": 5, + "docs.totaljs.com": 5, + "docs.velociraptor.app": 5, + "docs.veracode.com": 5, + "docs.vyperlang.org": 5, + "docs.wagtail.io": 5, + "documentation.concrete5.org": 5, + "documentation.wazuh.com": 5, + "doi.org": 5, + "dotnetnuke.codeplex.com": 5, + "dubell.io": 5, + "dustri.org": 5, + "dwrensha.github.io": 5, + "security.360.cn": 5, + "security.openttd.org": 5, + "securitywarrior9.blogspot.com": 5, + "edg.io": 5, + "edhunter484.medium.com": 5, + "eldstal.se": 5, + "electron.atom.io": 5, + "elgg.org": 5, + "emily.id.au": 5, + "en.bandisoft.com": 5, + "en.bitcoin.it": 5, + "engindemirbilek.github.io": 5, + "en.osdn.jp": 5, + "epadillas.github.io": 5, + "erlef.github.io": 5, + "eslam.io": 5, + "exceptionfactory.com": 5, + "experienceleague.adobe.com": 5, + "expressjs.com": 5, + "fastapi-admin-pro.long2ice.io": 5, + "fbdhhhh47.github.io": 5, + "febin0x4e4a.blogspot.com": 5, + "febin0x4e4a.wordpress.com": 5, + "febinj.medium.com": 5, + "ferrous-systems.com": 5, + "fgsec.net": 5, + "filezilla-project.org": 5, + "fisheye6.atlassian.com": 5, + "flask-limiter.readthedocs.io": 5, + "floqast.com": 5, + "flowiseai.com": 5, + "flyd.uk": 5, + "fmyyy1.github.io": 5, + "forge.centreon.com": 5, + "forge.typo3.org": 5, + "forum.codeigniter.com": 5, + "forum.datomic.com": 5, + "forum.ksec.co.uk": 5, + "forum.netgate.com": 5, + "forum.openzeppelin.com": 5, + "forums.couchbase.com": 5, + "forum.silverstripe.org": 5, + "forums.swift.org": 5, + "fossies.org": 5, + "freemarker.apache.org": 5, + "frycos.github.io": 5, + "fuo.fi": 5, + "fusionauth.io": 5, + "gainsec.com": 5, + "gauravnarwani.com": 5, + "genix.me": 5, + "geoserver.org": 5, + "gerrit.asterisk.org": 5, + "gerrit.ovirt.org": 5, + "getcomposer.org": 5, + "getgrav.org": 5, + "getkirby.com": 5, + "getlaminas.org": 5, + "getlektor.com": 5, + "git-annex.branchable.com": 5, + "git.fedorahosted.org": 5, + "github.com.mattermost": 5, + "git.libssh.org": 5, + "git.oschina.net": 5, + "git.reviewboard.kde.org": 5, + "git.sdaoden.eu": 5, + "git.xiph.org": 5, + "gld.mcphail.uk": 5, + "glitch.com": 5, + "gluu.org": 5, + "go2docs.graylog.org": 5, + "godoc.org": 5, + "gohugo.io": 5, + "go-review.git.corp.google.com": 5, + "graz.pure.elsevier.com": 5, + "greysec.net": 5, + "groups.drupal.org": 5, + "grumpz.net": 5, + "guides.spreecommerce.org": 5, + "gultsch.de": 5, + "hackerdna.com": 5, + "hacker.soarescorp.com": 5, + "hackers.report": 5, + "hackinglab.cz": 5, + "hackmysystems.tumblr.com": 5, + "hackpuntes.com": 5, + "hansmi.ch": 5, + "hapifhir.io": 5, + "help.ecostruxureit.com": 5, + "help.egroupware.org": 5, + "help.mulesoft.com": 5, + "help.panic.com": 5, + "help.passbolt.com": 5, + "help.rapid7.com": 5, + "help.sonatype.com": 5, + "shindig.apache.org": 5, + "shiro.apache.org": 5, + "homakov.blogspot.ru": 5, + "html5lib.readthedocs.io": 5, + "httpwg.org": 5, + "hub.docker.com": 5, + "hugegraph.apache.org": 5, + "i.blackhat.com": 5, + "ibm.github.io": 5, + "igniterealtime.atlassian.net": 5, + "silverpeas.com": 5, + "imgur.com": 5, + "infosec.exchange": 5, + "infra.spec.whatwg.org": 5, + "inhann.top": 5, + "inlong.apache.org": 5, + "insinuator.net": 5, + "intrix.com.au": 5, + "ipython.org": 5, + "ipython.readthedocs.io": 5, + "irrd.readthedocs.io": 5, + "isec.pl": 5, + "issues.ibexa.co": 5, + "issues.jasig.org": 5, + "issues.jenkins.io": 5, + "issues.sonatype.org": 5, + "it-sec.de": 5, + "itsmeanonartist.tech": 5, + "jadaptive.com": 5, + "janino-compiler.github.io": 5, + "java.net": 5, + "jay-from-future.github.io": 5, + "jenkins-ci.org": 5, + "jetpack.com": 5, + "jinja.palletsprojects.com": 5, + "jinmu1108.github.io": 5, + "jira.onosproject.org": 5, + "jira.opendaylight.org": 5, + "jira.sonarsource.com": 5, + "jira.whamcloud.com": 5, + "jquery.com": 5, + "jqueryui.com": 5, + "jqueryvalidation.org": 5, + "jsfiddle.net": 5, + "jub0bs.com": 5, + "junit.org": 5, + "kafka.apache.org": 5, + "kb.hitcon.org": 5, + "kb.netapp.com": 5, + "skipper.com": 5, + "koji.fedoraproject.org": 5, + "korelogic.com": 5, + "koz.io": 5, + "labanskoller.se": 5, + "labs.detectify.com": 5, + "lab.wallarm.com": 5, + "landave.io": 5, + "laravel-admin.org": 5, + "latestpcsolution.wordpress.com": 5, + "launchpad.support.sap.com": 5, + "laworigin.github.io": 5, + "layui.dev": 5, + "learn.snyk.io": 5, + "lednerb.de": 5, + "lessonsec.com": 5, + "libgit2.org": 5, + "libnmap.readthedocs.io": 5, + "lib.rs": 5, + "lightning.network": 5, + "limpidsecurity.pl": 5, + "linkerd.io": 5, + "linotp.org": 5, + "lists.cncf.io": 5, + "lists.denx.de": 5, + "lists.ffmpeg.org": 5, + "lists.mindrot.org": 5, + "lists.osgeo.org": 5, + "lists.w3.org": 5, + "lists.zx2c4.com": 5, + "locutus.io": 5, + "lutrasecurity.com": 5, + "lycshub.github.io": 5, + "m3n0sd0n4ld.github.io": 5, + "mail.gnome.org": 5, + "mailman-mail5.webfaction.com": 5, + "smalruby.jp": 5, + "marketplace.atlassian.com": 5, + "martinthomson.github.io": 5, + "mat4mee.notion.site": 5, + "matt.ucc.asn.au": 5, + "mayaseven.com": 5, + "media.dedaub.com": 5, + "mensfeld.pl": 5, + "meshery.io": 5, + "meta.wikimedia.org": 5, + "meterpreter.org": 5, + "minhnq22.medium.com": 5, + "mitmproxy.org": 5, + "mitogen.networkgenomics.com": 5, + "mksec.tk": 5, + "mlflow.org": 5, + "modwsgi.readthedocs.io": 5, + "monicz.dev": 5, + "morehouse.github.io": 5, + "mostwanted002.cf": 5, + "mouha.be": 5, + "movermeyer.com": 5, + "msrc-blog.microsoft.com": 5, + "mthbernardes.github.io": 5, + "mulch.dev": 5, + "my.f5.com": 5, + "my.goanywhere.com": 5, + "nablarch.atlassian.net": 5, + "nakedsecurity.sophos.com": 5, + "nandynarwhals.org": 5, + "narrow-oatmeal-0c0.notion.site": 5, + "nasa.github.io": 5, + "nest.pijul.com": 5, + "nghttp2.org": 5, + "nmap.org": 5, + "nodemailer.com": 5, + "node-postgres.com": 5, + "nostarttls.secvuln.info": 5, + "notes.netbytesec.com": 5, + "note.youdao.com": 5, + "note.zhaoj.in": 5, + "nova.app": 5, + "nowotarski.info": 5, + "nozero.io": 5, + "nsfocusglobal.com": 5, + "numanozdemir.com": 5, + "objectcomputing.com": 5, + "octoprint.org": 5, + "okankurtulus.com.tr": 5, + "omespino.com": 5, + "onekey.com": 5, + "openai.com": 5, + "openbase.com": 5, + "opencast.jira.com": 5, + "opencirt.com": 5, + "openjdk.org": 5, + "opennms.atlassian.net": 5, + "opensearch.org": 5, + "opensource.fast-report.com": 5, + "openssf.org": 5, + "opentelemetry.io": 5, + "opsecx.com": 5, + "orc.apache.org": 5, + "oroinc.com": 5, + "oryx-embedded.com": 5, + "oss-security.openwall.narkive.com": 5, + "osv.dev": 5, + "owncloud.com": 5, + "packaging.python.org": 5, + "pagehelper.github.io": 5, + "panda002.hashnode.dev": 5, + "papers.mathyvanhoef.com": 5, + "sparkjava.com": 5, + "payatu.com": 5, + "peckshield.com": 5, + "pentesterlab.com": 5, + "people.kingsds.network": 5, + "petl.readthedocs.io": 5, + "piraeus.io": 5, + "plotly.com": 5, + "plugins.gradle.org": 5, + "plus.google.com": 5, + "pony7.fr": 5, + "popalltheshells.medium.com": 5, + "powerful-bulb-c36.notion.site": 5, + "prestosql.io": 5, + "pretalx.com": 5, + "prismjs.com": 5, + "privatebin.info": 5, + "programmer.help": 5, + "projects.duckcorp.org": 5, + "prophaze.com": 5, + "publicobject.com": 5, + "pubs.acs.org": 5, + "pugjs.org": 5, + "pulsar.apache.org": 5, + "pydio.com": 5, + "pythonhosted.org": 5, + "python-hyper.org": 5, + "quilljs.com": 5, + "quiltmc.org": 5, + "r0.haxors.org": 5, + "ranchermanager.docs.rancher.com": 5, + "rankmath.com": 5, + "rastating.github.io": 5, + "ratpack.io": 5, + "reportportal.io": 5, + "repo.saltproject.io": 5, + "researchgate.net": 5, + "research.hisolutions.com": 5, + "research.insecurelabs.org": 5, + "restsharp.dev": 5, + "review.whamcloud.com": 5, + "rhodecode.com": 5, + "rhynorater.github.io": 5, + "robertheaton.com": 5, + "rodelllemit.medium.com": 5, + "rootdaemon.com": 5, + "roumenpetrov.info": 5, + "rpyc.readthedocs.io": 5, + "ruby-doc.org": 5, + "rudnkh.me": 5, + "sahildhar.github.io": 5, + "samuzora.com": 5, + "saturncloud.io": 5, + "sca.analysiscenter.veracode.com": 5, + "scalyr-static.s3.amazonaws.com": 5, + "scikit-learn.org": 5, + "scnps.co": 5, + "scratch.mit.edu": 5, + "sec.stealthcopter.com": 5, + "secure1.securityspace.com": 5, + "security-garage.com": 5, + "security.gradle.com": 5, + "securitylabs.datadoghq.com": 5, + "security.paloaltonetworks.com": 5, + "securitypitfalls.wordpress.com": 5, + "securitytrails.com": 5, + "segment.com": 5, + "sektioneins.de": 5, + "sethmlarson.dev": 5, + "setuptools.pypa.io": 5, + "shattered.io": 5, + "sheetjs.com": 5, + "siebene.github.io": 5, + "silverpeas.org": 5, + "skalatan.de": 5, + "skerritt.blog": 5, + "skii.dev": 5, + "skywalking.apache.org": 5, + "sling.apache.org": 5, + "smshrimant.com": 5, + "snicco.io": 5, + "softwaresupport.hpe.com": 5, + "solidus.io": 5, + "sonarsource.atlassian.net": 5, + "sources.debian.net": 5, + "spacemesh.io": 5, + "specifications.freedesktop.org": 5, + "spec.matrix.org": 5, + "src.fedoraproject.org": 5, + "srcincite.io": 5, + "stackstorm.com": 5, + "staging-website.elastic.co": 5, + "stash.kopano.io": 5, + "subrion.org": 5, + "support.aerospike.com": 5, + "support.ca.com": 5, + "support.citrix.com": 5, + "support.confluent.io": 5, + "support.contrastsecurity.com": 5, + "support.delphix.com": 5, + "support.herodevs.com": 5, + "support.snyk.io": 5, + "sw.aveva.com": 5, + "sylabs.io": 5, + "sylius.com": 5, + "sympa-community.github.io": 5, + "sysdig.com": 5, + "systeminformation.io": 5, + "systemweakness.com": 5, + "tada.github.io": 5, + "tadayoshi-sato.medium.com": 5, + "tantosec.com": 5, + "tches.iacr.org": 5, + "techblog.wikimedia.org": 5, + "technet.microsoft.com": 5, + "tempered.works": 5, + "tf1t.gitbook.io": 5, + "thehackernews.com": 5, + "the-it-wonders.blogspot.com": 5, + "thelia.net": 5, + "threat.tevora.com": 5, + "threema.ch": 5, + "tickets.puppetlabs.com": 5, + "tiki.org": 5, + "stimulsoft.com": 5, + "tprynn.github.io": 5, + "tracker.die-offenbachs.homelinux.org": 5, + "tracker.moodle.org": 5, + "tracker.phpbb.com": 5, + "tracker.zkoss.org": 5, + "trac.xapian.org": 5, + "trafficcontrol.apache.org": 5, + "travis-ci.com": 5, + "trends.builtwith.com": 5, + "strimzi.com": 5, + "trino.io": 5, + "tristartom.github.io": 5, + "truedigitalsecurity.com": 5, + "trufflesecurity.com": 5, + "trungvm.gitbook.io": 5, + "trust.neo4j.com": 5, + "tryhexadecimal.com": 5, + "tttang.com": 5, + "unbound.net": 5, + "unpoly.com": 5, + "updates.playhive.com": 5, + "updates.snyk.io": 5, + "support.zeus.com": 5, + "suricata-ids.org": 5, + "us-cert.cisa.gov": 5, + "vapor.codes": 5, + "vega.github.io": 5, + "veracode.com": 5, + "verichains.io": 5, + "versprite.com": 5, + "vertx.io": 5, + "vimeo.com": 5, + "vitejs.dev": 5, + "svn.savannah.gnu.org": 5, + "vulners.com": 5, + "w3lib.readthedocs.io": 5, + "warehouse.python.org": 5, + "weakdh.org": 5, + "webassembly.github.io": 5, + "webiny.com": 5, + "webmasters.googleblog.com": 5, + "wger.de": 5, + "wicket.apache.org": 5, + "wiki.duraspace.org": 5, + "wiki.eclipse.org": 5, + "wiki.folio.org": 5, + "wiki.gentoo.org": 5, + "wiki.mercurial-scm.org": 5, + "wiki.opendaylight.org": 5, + "wiki.shibboleth.net": 5, + "winscp.net": 5, + "wintercms.com": 5, + "winterdragon.ca": 5, + "wiremock.org": 5, + "withknown.com": 5, + "wtfsec.org": 5, + "activecyber.us": 5, + "admidio.org": 5, + "adminer.org": 5, + "aleksey.com": 5, + "alevsk.com": 5, + "alluxio.io": 5, + "ansible.com": 5, + "anyscale.com": 5, + "apereo.org": 5, + "apolloconfig.com": 5, + "baeldung.com": 5, + "barracuda.com": 5, + "benthamsgaze.org": 5, + "bountysource.com": 5, + "brics.dk": 5, + "broadcom.com": 5, + "brzozowski.io": 5, + "bsi.bund.de": 5, + "ccsq8.com": 5, + "certik.com": 5, + "chtsecurity.com": 5, + "cigital.com": 5, + "ciphertechs.com": 5, + "cloudflare.com": 5, + "cloudfoundry.org": 5, + "coalfire.com": 5, + "concrete5.org": 5, + "coresecurity.com": 5, + "crowdstrike.com": 5, + "crushftp.com": 5, + "cvedetails.com": 5, + "cybereagle.io": 5, + "cyfirma.com": 5, + "darkreading.com": 5, + "ddosi.org": 5, + "descope.com": 5, + "dnspython.org": 5, + "docker.com": 5, + "doyler.net": 5, + "dsecbypass.com": 5, + "dubget.com": 5, + "economizzer.org": 5, + "edwardthomson.com": 5, + "elementary-data.com": 5, + "enhavo.com": 5, + "enterprisedb.com": 5, + "equalexperts.com": 5, + "eyecontrol.nl": 5, + "fastify.io": 5, + "ffmpeg.org": 5, + "fireeye.com": 5, + "fork-cms.com": 5, + "fossil-scm.org": 5, + "gitpod.io": 5, + "gradio.app": 5, + "gsma.com": 5, + "hacksecproject.com": 5, + "hackthebox.com": 5, + "hakaioffensivesecurity.com": 5, + "haproxy.com": 5, + "htmlunit.org": 5, + "ihteam.net": 5, + "imperialviolet.org": 5, + "impresscms.org": 5, + "intruder.io": 5, + "isc.org": 5, + "jetbrains.com": 5, + "jhipster.tech": 5, + "lancom-systems.de": 5, + "landaire.net": 5, + "laravel-enlightn.com": 5, + "limesurvey.org": 5, + "logicallysecure.com": 5, + "mageni.net": 5, + "makotemplates.org": 5, + "mandiant.com": 5, + "menlosecurity.com": 5, + "mgm-sp.com": 5, + "monolune.com": 5, + "mpg123.de": 5, + "nds.ruhr-uni-bochum.de": 5, + "netlify.com": 5, + "netsarang.com": 5, + "ni.com": 5, + "nomadproject.io": 5, + "nu11secur1ty.com": 5, + "obrela.com": 5, + "octobot.online": 5, + "okta.com": 5, + "on-x.com": 5, + "op-c.net": 5, + "opencart.com": 5, + "opencrx.org": 5, + "optiv.com": 5, + "oxeye.io": 5, + "papermerge.com": 5, + "paramiko.org": 5, + "pethuraj.com": 5, + "phoronix.com": 5, + "pizzapower.me": 5, + "portainer.io": 5, + "postfix.org": 5, + "www-prd-trops.events.ibm.com": 5, + "prestashop.com": 5, + "privacy-wise.com": 5, + "projectcalico.org": 5, + "purplemet.com": 5, + "qemu.org": 5, + "raifberkaydincel.com": 5, + "rejetto.com": 5, + "reportlab.com": 5, + "rsaconference.com": 5, + "rust-lang.org": 5, + "sap.com": 5, + "scottbrady91.com": 5, + "secureauth.com": 5, + "securitymetrics.com": 5, + "semantic-mediawiki.org": 5, + "shielder.com": 5, + "shorebreaksecurity.com": 5, + "sidertia.com": 5, + "slf4j.org": 5, + "sockjs.org": 5, + "splunk.com": 5, + "symantec.com": 5, + "sympa.org": 5, + "tcpdump.org": 5, + "telerik.com": 5, + "terrapin-attack.com": 5, + "trustmatta.com": 5, + "vandyke.com": 5, + "vdoo.com": 5, + "viralpatel.net": 5, + "virtualbox.org": 5, + "whitehats.nl": 5, + "winimage.com": 5, + "yubico.com": 5, + "zdnet.com": 5, + "zenml.io": 5, + "zofrex.com": 5, + "zsh.org": 5, + "xalan.apache.org": 5, + "xavibel.com": 5, + "xmit.xyz": 5, + "xmlgraphics.apache.org": 5, + "xmpp.org": 5, + "yondon.blog": 5, + "zerosecuritypenetrationtesting.com": 5, + "zh-cn.tenable.com": 5, + "zookeeper.apache.org": 5, + "zuso.ai": 5, + "zxsecurity.co.nz": 5, + "testh5shanglv.minshengec.com:1024": 5, + "thinkphp.com": 5, + "tiddlywiki5.com": 5, + "trac.dojotoolkit.org": 5, + "tracker.ceph.com": 5, + "tree-kit.com": 5, + "unomi.apache.org.": 5, + "ureport.com": 5, + "users.encs.concordia.ca": 5, + "vsintelli.com": 5, + "wiki.apache.org": 5, + "wiki.rpath.com": 5, + "andmp.com": 5, + "benjaminfleischer.com": 5, + "bigdiao.cc": 5, + "blcat.cn": 5, + "brocade.com": 5, + "conviso.com.br": 5, + "coreftp.com": 5, + "cs.utexas.edu": 5, + "dest-unreach.org": 5, + "egroupware.org": 5, + "elasticsearch.com": 5, + "fetchmail.info": 5, + "fomori.org": 5, + "getsymphony.com": 5, + "gevent.org": 5, + "graphicsmagick.org": 5, + "hackersb.cn": 5, + "inspircd.org": 5, + "isg.rhul.ac.uk": 5, + "ja-sig.org": 5, + "jcraft.com": 5, + "jianshu.com": 5, + "linuxgrill.com": 5, + "mbsd.jp": 5, + "mega-nerd.com": 5, + "movabletype.org": 5, + "mpxj.org": 5, + "nemux.org": 5, + "netbytesec.com": 5, + "opera.com": 5, + "pip-installer.org": 5, + "procheckup.com": 5, + "rabbitmq.com": 5, + "roothc.com.br": 5, + "rootlabs.com.br": 5, + "rsyslog.com": 5, + "securityweek.com": 5, + "slideshare.net": 5, + "sqlalchemy.org": 5, + "square16.org": 5, + "stack.nl": 5, + "tinc-vpn.org": 5, + "tripwire.com": 5, + "uzbl.org": 5, + "zeroscience.mk": 5, + "xmlsoft.org": 5, + "xxl-job.com": 5, + "yehg.net": 5, + "yourls.org": 5, + "zalando.com": 5, + "sony.com": 5, + "strawberry.rocks": 5, + "werewolves.world": 5, + "newegg.com": 5, + "consumer.huawei.com": 5, + "kb.parallels.com": 5, + "support.google.com": 5, + "community.targit.com": 5, + "unisoc.com": 5, + "support.broadcom.com": 5, + "security-portal.versa-networks.com": 5, + "sliderrevolution.com": 5, + "autodesk.com": 5, + "trust.mi.com": 5, + "git.proxmox.com": 5, + "docs.azul.com": 5, + "kibty.town": 5, + "flowus.cn": 5, + "wp-osm-plugin.hyumika.com": 5, + "nokia.com": 5, + "bdosecurity.de": 5, + "khronokernel.com": 5, + "wiki.securepoint.de": 5, + "bridewell.com": 5, + "arc.net": 5, + "bravurasecurity.com": 5, + "chiggerlor.substack.com": 5, + "csflabs.github.io": 5, + "manageengine.com": 5, + "zoho.com": 5, + "devolutions.net": 5, + "technitium.com": 5, + "dell.com": 5, + "eviden.com": 5, + "ptzoptics.com": 5, + "security.freebsd.org": 5, + "giflib.com": 5, + "support.hp.com": 5, + "icecms.com": 5, + "cyberdanube.com": 5, + "customers.codesys.com": 5, + "kaiten.ru": 5, + "keyence.com": 5, + "trustedcare.entrust.com": 5, + "codefactor.io": 5, + "codeastro.com": 5, + "exchange.checkmk.com": 5, + "cs-cart.com": 5, + "usvn.info": 5, + "takex-eng.co.jp": 5, + "ti.qianxin.com": 5, + "stuxxn.github.io": 5, + "trevorkems.com": 5, + "cve.mahi.be": 5, + "0xmupa.github.io": 5, + "forum.proxmox.com": 5, + "puredata.info": 5, + "docs.rocket.chat": 5, + "cirosec.de": 5, + "docs.adacore.com": 5, + "forums.ivanti.com": 5, + "thewatch.centreon.com": 5, + "papercut.com": 5, + "sophos.com": 5, + "issue-tracker.miraheze.org": 5, + "tiptel.com": 5, + "e-tax.nta.go.jp": 5, + "teamviewer.com": 5, + "answers.webroot.com": 5, + "cert.vde.com": 5, + "support.bull.com": 5, + "cnnvd.org.cn": 5, + "westerndigital.com": 5, + "closed-loop.biz": 5, + "entrust.com": 5, + "solvait.com": 5, + "woodwing.com": 5, + "blog.hawktesters.com": 5, + "zyxel.com": 5, + "0xahmed.ninja": 5, + "2012.appsec-forum.ch": 5, + "acme.com": 5, + "aliyundrive-webdav.com": 5, + "als.regnet.cz": 5, + "aluigi.altervista.org": 5, + "antirez.com": 5, + "apt.inguza.net": 5, + "arthurdejong.org": 5, + "asfws12.files.wordpress.com": 5, + "askubuntu.com": 5, + "aspell.net": 5, + "badlock.org": 5, + "baraktawily.blogspot.com": 5, + "benmmurphy.github.io": 5, + "bh.ht.vc": 5, + "bird.network.cz": 5, + "bk.ntp.org": 5, + "blog.addepar.com": 5, + "blog.c22.cc": 5, + "blog.chromium.org": 5, + "blog.dscpl.com.au": 5, + "blog.fortify.com": 5, + "blog.fuseyism.com": 5, + "blog.gmane.org": 5, + "blog.g-sec.lu": 5, + "blog.guya.net": 5, + "blog.infosectcbr.com.au": 5, + "blog.kazuhooku.com": 5, + "blog.koehntopp.de": 5, + "blog.nibblesec.org": 5, + "blog.noobroot.com": 5, + "blog.o0o.nu": 5, + "blog.opensecurityresearch.com": 5, + "blog.pear.php.net": 5, + "blog.pi3.com.pl": 5, + "blog.prosody.im": 5, + "blog.python.org": 5, + "blogs.adobe.com": 5, + "blog.scrt.ch": 5, + "blog.securitymouse.com": 5, + "blog.sendsafely.com": 5, + "blog.senr.io": 5, + "blogs.iss.net": 5, + "blogs.oracle.com": 5, + "blog.spiderlabs.com": 5, + "blogs.sun.com": 5, + "blog.swiecki.net": 5, + "blog.swiftmailer.org": 5, + "blog.topsec.com.cn": 5, + "boinc.berkeley.edu": 5, + "bugs.dokuwiki.org": 5, + "bugs.gw.com": 5, + "bugs.icu-project.org": 5, + "bugs.java.com": 5, + "bugs.ntp.org": 5, + "bugs.openttd.org": 5, + "bugzilla.maptools.org": 5, + "buzz.typo3.org": 5, + "cakeforge.org": 5, + "cat.eyalro.net": 5, + "checkpw.sourceforge.net": 5, + "cisofy.com": 5, + "clicky.me": 5, + "co3k.org": 5, + "code.fabfile.org": 5, + "code.qt.io": 5, + "community.igniterealtime.org": 5, + "crypto.junod.info": 5, + "csrf.htmlpurifier.org": 5, + "cve.circl.lu": 5, + "cvs.sourceforge.net": 5, + "cybersecurity.upv.es": 5, + "danlec.com": 5, + "dev.exiv2.org": 5, + "dev.mutt.org": 5, + "dev.plone.org": 5, + "dev.subrion.org": 5, + "distro.conectiva.com.br": 5, + "docs.sequelizejs.com": 5, + "download.opensuse.org": 5, + "download.osgeo.org": 5, + "download.savannah.gnu.org": 5, + "download.strongswan.org": 5, + "drosenbe.blogspot.com": 5, + "drupalcode.org": 5, + "dsecrg.com": 5, + "em386.blogspot.com": 5, + "everdox.net": 5, + "fastcompression.blogspot.fr": 5, + "flash.flowplayer.org": 5, + "forum.xda-developers.com": 5, + "freeciv.wikia.com": 5, + "ftp.gnu.org": 5, + "ftp.naist.jp": 5, + "ftp.netbsd.org": 5, + "ftp.NetBSD.org": 5, + "gaganpreet.in": 5, + "ganglia.info": 5, + "geojson.org": 5, + "getid3.sourceforge.net": 5, + "git-blame.blogspot.com.es": 5, + "git.cyrusimap.org": 5, + "git.gluster.org": 5, + "git.hylafax.org": 5, + "git.infradead.org": 5, + "git.kernel.dk": 5, + "git.libav.org": 5, + "git.lxde.org": 5, + "git.mathias-kettner.de": 5, + "git.nordu.net": 5, + "gitorious.org": 5, + "git.savannah.nongnu.org": 5, + "git.tuxfamily.org": 5, + "googlechromereleases.blogspot.co.uk": 5, + "gridscheduler.sourceforge.net": 5, + "guac-dev.org": 5, + "h30499.www3.hp.com": 5, + "habrahabr.ru": 5, + "hac425.unaux.com": 5, + "ha.xxor.se": 5, + "hg.libsdl.org": 5, + "hg.savannah.gnu.org": 5, + "hkpco.kr": 5, + "homakov.blogspot.com": 5, + "icedtea.classpath.org": 5, + "inertiawar.com": 5, + "invisible-island.net": 5, + "ircrash.com": 5, + "isecpartners.github.io": 5, + "jaanuskp.blogspot.com": 5, + "jgarber.lighthouseapp.com": 5, + "jira.opensymphony.com": 5, + "jira.ow2.org": 5, + "juliusdavies.ca": 5, + "k3research.outerhaven.de": 5, + "kbase.redhat.com": 5, + "kerneltrap.org": 5, + "klikki.fi": 5, + "knoxin.blogspot.co.uk": 5, + "krbdev.mit.edu": 5, + "lab.cs.ttu.ee": 5, + "lcamtuf.blogspot.com.au": 5, + "ledgersmbdev.blogspot.ca": 5, + "libcloud.apache.org": 5, + "libexif.sourceforge.net": 5, + "linuxtesting.org": 5, + "linuxtv.org": 5, + "listengine.tuxfamily.org": 5, + "lists.alioth.debian.org": 5, + "lists.busybox.net": 5, + "lists.clamav.net": 5, + "lists.fusionforge.org": 5, + "lists.infradead.org": 5, + "lists.llvm.org": 5, + "lists.ntp.org": 5, + "lists.openvz.org": 5, + "lists.owasp.org": 5, + "lists.unbit.it": 5, + "lists.vmware.com": 5, + "lists.xiph.org": 5, + "lists.xymon.com": 5, + "lucene.apache.org": 5, + "lustre.org": 5, + "mahara.org": 5, + "mail.globnix.net": 5, + "mail.jabber.org": 5, + "mail.kde.org": 5, + "mailman.alsa-project.org": 5, + "mailman.mit.edu": 5, + "mail.openjdk.java.net": 5, + "mails.dpdk.org": 5, + "malloc.im": 5, + "martin.swende.se": 5, + "mathias-kettner.de": 5, + "mayaa.seasar.org": 5, + "metadata.ftp-master.debian.org": 5, + "mirror.easyname.at": 5, + "mirror.linux.org.au": 5, + "mod-security.svn.sourceforge.net": 5, + "modwsgi.readthedocs.org": 5, + "moinmoin.wikiwikiweb.de": 5, + "mov.sx": 5, + "mstrokin.com": 5, + "mysqlblog.fivefarmers.com": 5, + "netatalk.sourceforge.net": 5, + "net-ninja-mr.me": 5, + "news.dieweltistgarnichtso.net": 5, + "news.tryton.org": 5, + "nmav.gnutls.org": 5, + "nongnu.askapache.com": 5, + "old.blog.phusion.nl": 5, + "openjpeg.googlecode.com": 5, + "openvswitch.org": 5, + "osandamalith.wordpress.com": 5, + "osdir.com": 5, + "oss.sgi.com": 5, + "otiose.dhs.org": 5, + "owasp-esapi-java.googlecode.com": 5, + "owasp-java-html-sanitizer.googlecode.com": 5, + "ozlabs.org": 5, + "packetstormsecurity.nl": 5, + "pastie.caboo.se": 5, + "penturalabs.wordpress.com": 5, + "people.debian.org": 5, + "perception-point.io": 5, + "perl5.git.perl.org": 5, + "phpunit.vulnbusters.com": 5, + "pillow.readthedocs.org": 5, + "pmt.sourceforge.net": 5, + "post-office.corp.redhat.com": 5, + "proftpd.org": 5, + "projects.edgewall.com": 5, + "pseudo-flaw.net": 5, + "pylonshq.com": 5, + "pyropus.ca": 5, + "pywebdav.googlecode.com": 5, + "quassel-irc.org": 5, + "quickgit.kde.org": 5, + "qwertwwwe.github.io": 5, + "r00tin.blogspot.com": 5, + "rabbit.dereferenced.org": 5, + "rack.github.com": 5, + "railspikes.com": 5, + "restlet.org": 5, + "riddle.link": 5, + "rosariosis.com": 5, + "roundup.cvs.sourceforge.net": 5, + "rubysec.github.io": 5, + "01.org": 5, + "0day.work": 5, + "s1m0n.dft-labs.eu": 5, + "admin.hostpoint.ch": 5, + "alioth-lists.debian.net": 5, + "android-review.googlesource.com": 5, + "anotepad.com": 5, + "antichat.com": 5, + "appgateresearch.blogspot.com": 5, + "archive.apache.org": 5, + "artifex.com": 5, + "attachments.samba.org": 5, + "attic.apache.org": 5, + "bertjwregeer.keybase.pub": 5, + "bitslog.com": 5, + "blade.tencent.com": 5, + "blog.cloudpassage.com": 5, + "blog.convisoappsec.com": 5, + "blog.cryptographyengineering.com": 5, + "blog.documentfoundation.org": 5, + "blog.hartwork.org": 5, + "blog.hboeck.de": 5, + "blog.mozilla.org": 5, + "blog.qt.io": 5, + "blog.quarkslab.com": 5, + "blog.smarttecs.com": 5, + "blog.sqreen.com": 5, + "blog.wpscan.org": 5, + "bnbdr.github.io": 5, + "botan.randombit.net": 5, + "bpmn.io": 5, + "bro-tracker.atlassian.net": 5, + "bugreports.qt.io": 5, + "bugs.exim.org": 5, + "bugs.gnunet.org": 5, + "bugs.horde.org": 5, + "bugs.linuxfoundation.org": 5, + "bugs.openvz.org": 5, + "bugs.webkit.org": 5, + "bugzilla.clamav.net": 5, + "bugzilla.mindrot.org": 5, + "bugzilla.samba.org": 5, + "bugzilla.tianocore.org": 5, + "busybox.net": 5, + "c-ares.haxx.se": 5, + "ceph.io": 5, + "cfreal.github.io": 5, + "chromiumcodereview.appspot.com": 5, + "civicrm.org": 5, + "codechecker-demo.eastus.cloudapp.azure.com": 5, + "code.launchpad.net": 5, + "codereview.appspot.com": 5, + "codesearch.debian.net": 5, + "community.jboss.org": 5, + "community.qualys.com": 5, + "community.sophos.com": 5, + "contrib.spip.net": 5, + "coreymhudson.github.io": 5, + "crashes.fuzzing-project.org": 5, + "crrev.com": 5, + "crypto.stanford.edu": 5, + "defuse.ca": 5, + "deshal3v.github.io": 5, + "developer.atlassian.com": 5, + "dev.gnupg.org": 5, + "dev.recurly.com": 5, + "discourse.aurelia.io": 5, + "discourse.gnome.org": 5, + "discuss.gradle.org": 5, + "discuss.neos.io": 5, + "docs.inspircd.org": 5, + "dom4j.github.io": 5, + "donncha.is": 5, + "download.igniterealtime.org": 5, + "dyntopia.com": 5, + "secondlookforensics.com": 5, + "secureappdev.blogspot.com": 5, + "secureyourit.co.uk": 5, + "security.cucumberlinux.com": 5, + "edk2-docs.gitbook.io": 5, + "edk2.groups.io": 5, + "edoverflow.com": 5, + "efail.de": 5, + "elixir.bootlin.com": 5, + "en.bitcoinwiki.org": 5, + "enigmail.net": 5, + "esnet-security.github.io": 5, + "exploitbox.io": 5, + "eyalitkin.wordpress.com": 5, + "fakhrizulkifli.github.io": 5, + "fastcompression.blogspot.ca": 5, + "fastd.readthedocs.io": 5, + "feh.finalrewind.org": 5, + "fisheye.codehaus.org": 5, + "forums.grsecurity.net": 5, + "forum.snapcraft.io": 5, + "forums.servicestack.net": 5, + "francozappa.github.io": 5, + "sf.snu.ac.kr": 5, + "fuelphp.com": 5, + "gaffer.ptitcanardnoir.org": 5, + "gerrit.googlesource.com": 5, + "git.busybox.net": 5, + "git.centos.org": 5, + "git.entrouvert.org": 5, + "git.gnunet.org": 5, + "git.gnupg.org": 5, + "github.blog": 5, + "githubengineering.com": 5, + "gitlab.xiph.org": 5, + "git.netfilter.org": 5, + "git.tartarus.org": 5, + "git.tt-rss.org": 5, + "git.whamcloud.com": 5, + "git.zabbix.com": 5, + "gnats.netbsd.org": 5, + "gnupg.org": 5, + "googleonlinesecurity.blogspot.com": 5, + "googleprojectzero.blogspot.co.uk": 5, + "googleprojectzero.blogspot.cz": 5, + "gruss.cc": 5, + "guidovranken.files.wordpress.com": 5, + "gusralph.info": 5, + "h20564.www2.hpe.com": 5, + "h20566.www2.hp.com": 5, + "heasarc.gsfc.nasa.gov": 5, + "hg.ucc.asn.au": 5, + "shibboleth.internet2.edu": 5, + "hosein-vita.medium.com": 5, + "httpoxy.org": 5, + "icepng.github.io": 5, + "icinga.com": 5, + "insert-script.blogspot.com": 5, + "insights.sei.cmu.edu": 5, + "insights.ubuntu.com": 5, + "invent.kde.org": 5, + "ioquake3.org": 5, + "issues.civicrm.org": 5, + "issues.gerritcodereview.com": 5, + "issuetracker.google.com": 5, + "site.icu-project.org": 5, + "site.pi3.com.pl": 5, + "jeffhacks.com": 5, + "kate.io": 5, + "kb.bluecoat.com": 5, + "kingcope.wordpress.com": 5, + "knobattack.com": 5, + "komodoplatform.com": 5, + "leastauthority.com": 5, + "libexpat.github.io": 5, + "libosinfo.org": 5, + "libreswan.org": 5, + "lists.ath9k.org": 5, + "lists.clusterlabs.org": 5, + "lists.cypherpunks.ca": 5, + "lists.iai.uni-bonn.de": 5, + "lists.immunityinc.com": 5, + "lists.isc.org": 5, + "lists.linuxcontainers.org": 5, + "lists.nic.cz": 5, + "lists.openldap.org": 5, + "lists.open-mesh.org": 5, + "lists.osuosl.org": 5, + "lists.sequoia-pgp.org": 5, + "lists.tartarus.org": 5, + "lists.ubuntu.com": 5, + "lock.cmpxchg8b.com": 5, + "mailarchives.bentasker.co.uk": 5, + "mail.gnu.org": 5, + "mailman-eng.corp.redhat.com": 5, + "make.wordpress.org": 5, + "mariadb.atlassian.net": 5, + "marlam.de": 5, + "smarty-php.googlecode.com": 5, + "maustin.net": 5, + "mcabber.com": 5, + "mega.nz": 5, + "meltdownattack.com": 5, + "micahflee.com": 5, + "mirror.fail": 5, + "nathandavison.com": 5, + "neomutt.org": 5, + "neopg.io": 5, + "nethack.org": 5, + "newrelic.com": 5, + "nohats.ca": 5, + "snoopy.cvs.sourceforge.net": 5, + "nvisium.com": 5, + "old.reddit.com": 5, + "open-docs.neuvector.com": 5, + "openmpt.org": 5, + "openssf.slack.com": 5, + "openssl-library.org": 5, + "opnsec.com": 5, + "orpheus-lyre.info": 5, + "osdn.net": 5, + "oss.clusterlabs.org": 5, + "os-s.net": 5, + "ostif.org": 5, + "source.jboss.org": 5, + "packages.qa.debian.org": 5, + "paste.pound-python.org": 5, + "patchwork.ffmpeg.org": 5, + "patchwork.freedesktop.org": 5, + "people.csail.mit.edu": 5, + "people.fedoraproject.org": 5, + "phabricator.kde.org": 5, + "philwantsfish.github.io": 5, + "spl0it.org": 5, + "platypusattack.com": 5, + "postlister.uninett.no": 5, + "projects.gnome.org": 5, + "projects.ow2.org": 5, + "pyyaml.docsforge.com": 5, + "qa.debian.org": 5, + "qtpass.org": 5, + "rachelbythebay.com": 5, + "raw.github.com": 5, + "raw.globalsecuritydatabase.org": 5, + "rdot.org": 5, + "relistan.com": 5, + "robots.thoughtbot.com": 5, + "rtpbleed.com": 5, + "rt.perl.org": 5, + "ruhr-uni-bochum.sciebo.de": 5, + "scannell.me": 5, + "scarybeastsecurity.blogspot.ch": 5, + "scarybeastsecurity.blogspot.com": 5, + "scarybeastsecurity.blogspot.dk": 5, + "scumjr.github.io": 5, + "secure.phabricator.com": 5, + "secure.php.net": 5, + "security-center.intel.com": 5, + "serenity.is": 5, + "sick.codes": 5, + "sitewat.ch": 5, + "smacktls.com": 5, + "source.openmpt.org": 5, + "spectreattack.com": 5, + "speirofr.appspot.com": 5, + "ssrg.nicta.com.au": 5, + "support.cloud.engineyard.com": 5, + "support.process-one.net": 5, + "svn.filezilla-project.org": 5, + "svn.resiprocate.org": 5, + "sweet32.info": 5, + "swiftstack.com": 5, + "sword.bladex.cn": 5, + "sympa.inria.fr": 5, + "t2.fi": 5, + "tapestry.apache.org": 5, + "thejh.net": 5, + "therecord.media": 5, + "threatpost.com": 5, + "tomforb.es": 5, + "trac.gajim.org": 5, + "tracker.ardour.org": 5, + "trac.mplayerhq.hu": 5, + "trac.nginx.org": 5, + "tt-rss.org": 5, + "unicode-org.atlassian.net": 5, + "unix.stackexchange.com": 5, + "utcc.utoronto.ca": 5, + "vishnudevtj.github.io": 5, + "vndh.net": 5, + "svn.ec-cube.net": 5, + "svn.haxx.se": 5, + "svn.ruby-lang.org": 5, + "svn.tartarus.org": 5, + "vulnhive.com": 5, + "web-in-security.blogspot.ca": 5, + "wid.cert-bund.de": 5, + "wiki.openvz.org": 5, + "wiki.samba.org": 5, + "worthdoingbadly.com": 5, + "wpa3.mathyvanhoef.com": 5, + "131002.net": 5, + "www3.sqlite.org": 5, + "www3.trustwave.com": 5, + "acunetix.com": 5, + "alphabot.com": 5, + "amd.com": 5, + "arista.com": 5, + "armis.com": 5, + "bamsoftware.com": 5, + "bitdefender.com": 5, + "cabextract.org.uk": 5, + "cazzulino.com": 5, + "cisecurity.org": 5, + "cl.cam.ac.uk": 5, + "cncf.io": 5, + "contextis.com": 5, + "cubeyond.net": 5, + "davical.org": 5, + "eclypsium.com": 5, + "evonide.com": 5, + "exiv2.org": 5, + "flexera.com": 5, + "freeplane.org": 5, + "futureweb.at": 5, + "gnu.org": 5, + "icinga.org": 5, + "inputzero.io": 5, + "intezer.com": 5, + "java.com": 5, + "jsof-tech.com": 5, + "jwz.org": 5, + "kernelmode.blog": 5, + "krackattacks.com": 5, + "kvakil.me": 5, + "ldap-account-manager.org": 5, + "legacysecuritygroup.com": 5, + "libssh2.org": 5, + "lsexperts.de": 5, + "mehmetince.net": 5, + "midnight-commander.org": 5, + "nikhef.nl": 5, + "npmjs.org": 5, + "openldap.org": 5, + "percona.com": 5, + "portcullis-security.com": 5, + "pureftpd.org": 5, + "python.org": 5, + "qbittorrent.org": 5, + "redteam-pentesting.de": 5, + "reversinglabs.com": 5, + "rootshellsecurity.net": 5, + "rorsecurity.info": 5, + "saddns.net": 5, + "secfu.net": 5, + "secura.com": 5, + "seebug.org": 5, + "sentinelone.com": 5, + "sigsac.org": 5, + "stunnel.org": 5, + "teskalabs.com": 5, + "theregister.co.uk": 5, + "thezdi.com": 5, + "timmclean.net": 5, + "trendmicro.com": 5, + "vice.com": 5, + "vidocsecurity.com": 5, + "viestintavirasto.fi": 5, + "virustotal.com": 5, + "vusec.net": 5, + "xairy.github.io": 5, + "sysoev.ru": 5, + "t3.dotgnu.info": 5, + "tartarus.org": 5, + "taviso.decsystem.org": 5, + "telia.dl.sourceforge.net": 5, + "telussecuritylabs.com": 5, + "thekelleys.org.uk": 5, + "tickets.opscode.com": 5, + "trac.kodi.tv": 5, + "trac.transmissionbt.com": 5, + "vcs.pcre.org": 5, + "venom.crowdstrike.com": 5, + "vladz.devzero.fr": 5, + "webservsec.blogspot.com": 5, + "wiki2.dovecot.org": 5, + "wiki.audacityteam.org": 5, + "wiki.lustre.org": 5, + "wiki.postgresql.org": 5, + "worldofpadman.com": 5, + "www13.itrc.hp.com": 5, + "agarri.fr": 5, + "agrs.tu-berlin.de": 5, + "apsis.ch": 5, + "argyllcms.com": 5, + "arm.linux.org.uk": 5, + "asmail.be": 5, + "attrition.org": 5, + "bacula.org": 5, + "betanews.com": 5, + "binarysniper.net": 5, + "bugzilla.org": 5, + "cc.gatech.edu": 5, + "cert.fi": 5, + "ciac.org": 5, + "cisco.com": 5, + "codenomicon.com": 5, + "collabtive.o-dyn.de": 5, + "communities.hp.com": 5, + "cs.bu.edu": 5, + "cs.tau.ac.IL": 5, + "cs.tau.ac.il": 5, + "cs.technion.ac.il": 5, + "dotnetnuke.com": 5, + "droid-life.com": 5, + "droidrzr.com": 5, + "droidsec.org": 5, + "educatedguesswork.org": 5, + "elasticsearch.org": 5, + "elsherei.com": 5, + "eucalyptus.com": 5, + "evernote.com": 5, + "exploringbinary.com": 5, + "firebirdsql.org": 5, + "floyd.ch": 5, + "freelists.org": 5, + "freerdp.com": 5, + "gdssecurity.com": 5, + "getchef.com": 5, + "gulftech.org": 5, + "gwtproject.org": 5, + "haproxy.org": 5, + "hdwsec.fr": 5, + "hitachi-support.com": 5, + "htbridge.ch": 5, + "ijg.org": 5, + "infoq.com": 5, + "infradead.org": 5, + "ingate.com": 5, + "intelsecurity.com": 5, + "jbkempf.com": 5, + "jplayer.org": 5, + "kernelhub.org": 5, + "konakart.com": 5, + "libraw.org": 5, + "live555.com": 5, + "lua.org": 5, + "mandrakesecure.net": 5, + "maradns.org": 5, + "mathyvanhoef.com": 5, + "metasploit.com": 5, + "mh-sec.de": 5, + "mindrot.org": 5, + "mitls.org": 5, + "mongodb.org": 5, + "mono-project.com": 5, + "nds.rub.de": 5, + "networkworld.com": 5, + "nntp.perl.org": 5, + "nruns.com": 5, + "nsfocus.net": 5, + "oliverkarow.de": 5, + "opencms.org": 5, + "openoffice.org": 5, + "palemoon.org": 5, + "paul-moore.com": 5, + "phenoelit.org": 5, + "pimcore.org": 5, + "pnigos.com": 5, + "process-one.net": 5, + "quantumleap.it": 5, + "rafayhackingarticles.net": 5, + "rajatswarup.com": 5, + "ratbox.org": 5, + "rul3z.de": 5, + "sbosnet.nl": 5, + "securation.com": 5, + "securegoose.org": 5, + "securiteam.com": 5, + "sendmail.com": 5, + "simplesystems.org": 5, + "slimframework.com": 5, + "sogo.nu": 5, + "splitbrain.org": 5, + "springsource.com": 5, + "squirrelmail.org": 5, + "supercluster.org": 5, + "synacktiv.ninja": 5, + "tedunangst.com": 5, + "tele-consulting.com": 5, + "tombom.co.uk": 5, + "tt-forums.net": 5, + "undeadly.org": 5, + "unixodbc.org": 5, + "x.org": 5, + "zweitag.de": 5, + "xstream.codehaus.org": 5, + "xteam.baidu.com": 5, + "xync.org": 5, + "zsh.sourceforge.net": 5, + "project-zero.issues.chromium.org": 5, + "anker.com": 5, + "visionspace.com": 5, + "eufy.com": 5, + "advisory-inbox.githubapp.com": 5, + "myoffice.ru": 5, + "support.myoffice.ru": 5, + "planex.co.jp": 5, + "docs.iredmail.org": 5, + "iredmail.org": 5, +} diff --git a/vulnerablecode-json-api.png b/vulnerablecode-json-api.png deleted file mode 100644 index 9712a9f93..000000000 Binary files a/vulnerablecode-json-api.png and /dev/null differ diff --git a/vulnerablecode-ui.png b/vulnerablecode-ui.png deleted file mode 100644 index 7a0a28329..000000000 Binary files a/vulnerablecode-ui.png and /dev/null differ diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index 22d19958f..ee339e883 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -3,16 +3,14 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os import sys -import warnings -from pathlib import Path -__version__ = "34.0.0rc5" +__version__ = "35.1.0" def command_line(): diff --git a/vulnerablecode/context_processors.py b/vulnerablecode/context_processors.py index e93bb7bd3..ee5885b81 100644 --- a/vulnerablecode/context_processors.py +++ b/vulnerablecode/context_processors.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 472c1405a..0e545e0f2 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -55,6 +55,8 @@ EMAIL_HOST_PASSWORD = env.str("EMAIL_HOST_PASSWORD", default="") FROM_EMAIL = env.str("FROM_EMAIL", default="") +VULNERABLECODE_LOG_LEVEL = env.str("VULNERABLECODE_LOG_LEVEL", "INFO") + # Application definition INSTALLED_APPS = ( @@ -76,6 +78,7 @@ "rest_framework.authtoken", "widget_tweaks", "crispy_forms", + "crispy_bootstrap4", # for API doc "drf_spectacular", # required for Django collectstatic discovery @@ -194,6 +197,7 @@ str(PROJECT_DIR / "static"), ] +CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap4" CRISPY_TEMPLATE_PACK = "bootstrap4" @@ -287,6 +291,9 @@ if DEBUG_TOOLBAR: + # Uncomment this to get pyinstrument profiles + # PYINSTRUMENT_PROFILE_DIR = "profiles" + INSTALLED_APPS += ("debug_toolbar",) MIDDLEWARE += ( @@ -314,3 +321,37 @@ INTERNAL_IPS = [ "127.0.0.1", ] + + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": { + "format": "{levelname} {message}", + "style": "{", + }, + }, + "handlers": { + "null": { + "class": "logging.NullHandler", + }, + "console": { + "class": "logging.StreamHandler", + "formatter": "simple", + }, + }, + "loggers": { + "vulnerabilities.pipelines": { + "handlers": ["console"], + "level": VULNERABLECODE_LOG_LEVEL, + "propagate": False, + }, + }, +} + +if DEBUG: + LOGGING["django"] = { + "handlers": ["console"], + "level": "ERROR", + } diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index a04427957..6d8918a8f 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -187,12 +187,6 @@ code { border-color: #dbdbdb; } -/* 2023-08-28 Monday 14:55:42. Is this still needed or does wrap-strings take its place? Keep eyes peeled for any odd displays. */ -/* .table td { - word-wrap: break-word; -} */ - - .wrap-strings { word-break: break-word; } @@ -356,10 +350,9 @@ a.small_page_button { } .details-container { - border: solid 1px #e8e8e8; border: 0; border-radius: 6px; - box-shadow: 0 0.5em 1em -0.125em rgb(10 10 10 / 10%), 0 0px 0 1px rgb(10 10 10 / 2%); + box-shadow: 0 0.5em 1em -0.125em rgba(8, 8, 8, 0.1), 0 0px 0 1px rgba(10, 10, 10, 0.02); } .about-hover-div { @@ -373,7 +366,7 @@ a.small_page_button { } span.tag.custom { - margin: 0px 0px 6px 10px; + margin: 0px 0px 0px 10px; } /* CSS for dev fixed by headers */ @@ -428,14 +421,9 @@ span.tag.custom { border: solid 1px #dbdbdb; background-color: #ffffff; } -/* test bulleted list */ ul.fixed_by_bullet { list-style-type: disc; - /*margin-top: 2px; -margin-bottom: 10px;*/ - /*margin-left: -24px;*/ - /*margin-left: -30px;*/ margin-top: 0.25em; margin-left: 7px; margin-bottom: 0.25em; @@ -444,11 +432,8 @@ margin-bottom: 10px;*/ ul.fixed_by_bullet ul { list-style-type: disc; - /*margin-top: 10px;*/ - margin-top: 5px; margin-top: 0px; margin-bottom: 0px; - margin-left: 23px; margin-left: 18px; padding: 0; border: none; @@ -472,7 +457,6 @@ ul.fixed_by_bullet li:last-child { font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; font-size: 13px; font-weight: normal; - /*margin-bottom: 10px;*/ margin-bottom: 0px; } @@ -486,12 +470,11 @@ ul.fixed_by_bullet li li { color: #000000; } -/* 10/10/15 add 3rd-level bullets */ +/* 3rd-level bullets */ ul.fixed_by_bullet ul ul { list-style-type: disc; margin-top: 0px; margin-bottom: 0px; - margin-left: 50px; margin-left: 17px; padding: 0; border: none; @@ -569,10 +552,106 @@ ul.fixed_by_bullet li li li { /* Emphasis for not vulnerable. */ .emphasis-not-vulnerable { background-color: #e6ffe6; - /* background-color: #e6ffff; */ + word-wrap: break-word; word-break: break-all; + display: block; } /* Emphasis for vulnerable. */ .emphasis-vulnerable { background-color: #ffe6e6; } + +/* From https://github.com/jgthms/bulma/issues/2040#issuecomment-734507270 (the Bulma GH repo under the author's top-level GH entity). This helps display the responsive navbar dropdown properly < 1024px width. JMH: some added styles to remove box-shadow and make other adjustnments for the collapsed navbar. */ +@media screen and (max-width: 1024px) { + .navbar-menu { + align-items: stretch; + background-color: transparent; + font-size: 0.875rem; + display: flex; + flex-grow: 1; + flex-shrink: 0; + padding: 0; + + margin-right: 0px !important; + + .navbar-item.is-active .navbar-dropdown, + .navbar-item.is-hoverable:focus .navbar-dropdown, + .navbar-item.is-hoverable:focus-within .navbar-dropdown, + .navbar-item.is-hoverable:hover .navbar-dropdown { + display: block; + } + + .navbar-end { + justify-content: flex-end; + margin-left: auto; + align-items: stretch; + display: flex; + + .navbar-item.has-dropdown { + align-items: stretch; + } + + .navbar-item, + .navbar-link { + align-items: center; + display: flex; + } + + .navbar-dropdown { + border-bottom-left-radius: 6px; + border-bottom-right-radius: 6px; + border-top: 2px solid #dbdbdb; + box-shadow: 0 8px 8px rgba(10, 10, 10, 0.1); + display: none; + font-size: 0.875rem; + left: 0; + min-width: 100%; + position: absolute; + top: 100%; + z-index: 20; + } + } + } + + /* Make sure the 'About' navbar link hovering div appears left-aligned on narrower screens so it's visible rather than forced beyond the left-hand screen edge. */ + .dropdown.is-right .dropdown-menu { + left: 0; + right: auto; + } + + /* Remove the collapsed menu's right-hand dark bar, inherited because of how Bulma handles this process. */ + div.navbar-end.mr-3 { + margin-right: 0 !important; + } + + .navbar-item, + .navbar-item:active, + .navbar-item:focus, + .navbar-item:visited { + color: #ffffff; + background-color: transparent; + align-items: stretch; + display: flex; + } + + div.navbar-start { + width: 100%; + } + + a.navbar-item:focus { + background-color: transparent; + color: #ffffff; + } + + a.navbar-item:hover { + color: #ffffff; + background-color: #000000; + } + +} + +@media screen and (max-width: 1023px) { + .navbar-menu { + box-shadow: none; + } +} diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 28954d3a8..10f7db13f 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -20,6 +20,8 @@ from vulnerabilities.api import CPEViewSet from vulnerabilities.api import PackageViewSet from vulnerabilities.api import VulnerabilityViewSet +from vulnerabilities.api_v2 import PackageV2ViewSet +from vulnerabilities.api_v2 import VulnerabilityV2ViewSet from vulnerabilities.views import ApiUserCreateView from vulnerabilities.views import HomePage from vulnerabilities.views import PackageDetails @@ -43,7 +45,12 @@ def __init__(self, *args, **kwargs): api_router.register("cpes", CPEViewSet, basename="cpe") api_router.register("aliases", AliasViewSet, basename="alias") +api_v2_router = OptionalSlashRouter() +api_v2_router.register("packages", PackageV2ViewSet, basename="package-v2") +api_v2_router.register("vulnerabilities", VulnerabilityV2ViewSet, basename="vulnerability-v2") + urlpatterns = [ + path("api/v2/", include(api_v2_router.urls)), path( "robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), diff --git a/vulnerablecode/wsgi.py b/vulnerablecode/wsgi.py index c17f4abeb..aa06c937a 100644 --- a/vulnerablecode/wsgi.py +++ b/vulnerablecode/wsgi.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulntotal/datasources/github.py b/vulntotal/datasources/github.py index 154a31fe6..57293f3cb 100644 --- a/vulntotal/datasources/github.py +++ b/vulntotal/datasources/github.py @@ -11,7 +11,7 @@ from typing import Iterable from dotenv import load_dotenv -from fetchcode.package_versions import github_response +from fetchcode.utils import github_response from packageurl import PackageURL from vulntotal.validator import DataSource