Ingest npm data through github api #1025

CHANGELOG.rst

@@ -10,6 +10,7 @@ Version v31.1.0
 - We are now handling purl fragments in package search. For example:
   you can now serch using queries in the UI like this : `[email protected]`,
   `cherrypy` or `pkg:pypi`.
+- We are now ingesting npm advisories data through GitHub API.
 
 
 Version v31.0.0

requirements.txt

@@ -54,7 +54,7 @@ MarkupSafe==2.1.1
 matplotlib-inline==0.1.3
 multidict==6.0.2
 mypy-extensions==0.4.3
-packageurl-python==0.10.3
+packageurl-python==0.10.5rc1
 packaging==21.3
 paramiko==2.10.3
 parso==0.8.3

setup.cfg

@@ -69,7 +69,7 @@ install_requires =
     coreapi>=2.3.3
 
     #essentials
-    packageurl-python>=0.9.4
+    packageurl-python>=0.10.5rc1
     univers>=30.9.0
     license-expression>=21.6.14
 

vulnerabilities/importers/github.py

@@ -114,6 +114,7 @@
     "COMPOSER": "composer",
     "PIP": "pypi",
     "RUBYGEMS": "gem",
+    "NPM": "npm",
     # "GO": "golang",
 }
 
@@ -122,8 +123,9 @@
 }
 
 # TODO: We will try to gather more info from GH API
+# Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885
 # Check https://github.com/nexB/vulnerablecode/issues/645
-# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI'}
+# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM'}
 # second '%s' is interesting, it will have the value '' for the first request,
 GRAPHQL_QUERY_TEMPLATE = """
 query{
@@ -202,13 +204,13 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]:
         ns, _, name = github_name.partition(":")
         return PackageURL(type=pkg_type, namespace=ns, name=name)
 
-    if pkg_type == "composer":
+    if pkg_type in ("composer", "npm"):
         if "/" not in github_name:
             return PackageURL(type=pkg_type, name=github_name)
         vendor, _, name = github_name.partition("/")
         return PackageURL(type=pkg_type, namespace=vendor, name=name)
 
-    if pkg_type in ("nuget", "pypi", "gem", "golang"):
+    if pkg_type in ("nuget", "pypi", "gem", "golang", "npm"):
         return PackageURL(type=pkg_type, name=github_name)
 
     logger.error(f"get_purl: Unknown package type {pkg_type}")

vulnerabilities/package_managers.py

@@ -266,7 +266,8 @@ class NpmVersionAPI(VersionAPI):
     package_type = "npm"
 
     def fetch(self, pkg):
-        url = f"https://registry.npmjs.org/{pkg}"
+        lower_pkg = pkg.lower()
+        url = f"https://registry.npmjs.org/{lower_pkg}"
         response = get_response(url=url, content_type="json")
         if not response:
             logger.error(f"Failed to fetch {url}")

vulnerabilities/tests/test_github.py

@@ -33,7 +33,7 @@
 TEST_DATA = os.path.join(BASE_DIR, "test_data", "github_api")
 
 
-@pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "golang", "composer", "pypi"])
+@pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "golang", "composer", "pypi", "npm"])
 def test_process_response_github_importer(pkg_type, regen=REGEN):
     response_file = os.path.join(TEST_DATA, f"{pkg_type}.json")
     expected_file = os.path.join(TEST_DATA, f"{pkg_type}-expected.json")