Full fledged processing of protection profiles

notebooks/cc/cpe_eval.ipynb

@@ -18,7 +18,8 @@
     "from sec_certs.dataset import CCDataset\n",
     "import pandas as pd\n",
     "import json\n",
-    "import tempfile"
+    "import tempfile\n",
+    "from sec_certs.utils.label_studio_utils import to_label_studio_json"
    ]
   },
   {
@@ -58,7 +59,7 @@
     "with tempfile.TemporaryDirectory() as tmp_dir:\n",
     "    dset.root_dir = tmp_dir\n",
     "    dset.certs = {x.dgst: x for x in dset if x.dgst in eval_certs.index.tolist()}\n",
-    "    dset.to_label_studio_json(\"./label_studio_input_data.json\", update_json=False)"
+    "    to_label_studio_json(dset, \"./label_studio_input_data.json\")"
    ]
   },
   {

notebooks/cc/scheme_eval.ipynb

@@ -24,7 +24,8 @@
     "from sec_certs.model import CCSchemeMatcher\n",
     "from sec_certs.sample.cc_certificate_id import canonicalize\n",
     "from sec_certs.sample.cc_scheme import CCScheme, EntryType\n",
-    "from sec_certs.configuration import config"
+    "from sec_certs.configuration import config\n",
+    "from sec_certs.dataset.auxiliary_dataset_handling import CCSchemeDatasetHandler"
    ]
   },
   {
@@ -56,7 +57,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dset.auxiliary_datasets.scheme_dset = schemes\n",
+    "dset.aux_handlers[CCSchemeDatasetHandler].dset = schemes\n",
     "\n",
     "count_was = 0\n",
     "count_is = 0\n",
@@ -161,7 +162,7 @@
     "        rate = len(assigned)/len(total) * 100 if len(total) != 0 else 0\n",
     "        rate_list = rates.setdefault(country, [])\n",
     "        rate_list.append(rate)\n",
-    "        \n",
+    "\n",
     "        print(f\"{country}: {len(assigned)} assigned out of {len(total)} -> {rate:.1f}%\")\n",
     "        total_active = total[total[\"status\"] == \"active\"]\n",
     "        assigned_active = assigned[assigned[\"status\"] == \"active\"]\n",

notebooks/cc/vulnerabilities.ipynb

@@ -33,6 +33,7 @@
     "import warnings\n",
     "from pathlib import Path\n",
     "import tempfile\n",
+    "from sec_certs.dataset.auxiliary_dataset_handling import CVEDatasetHandler, CPEDatasetHandler, CCMaintenanceUpdateDatasetHandler\n",
     "from sec_certs.dataset import CCDataset, CCDatasetMaintenanceUpdates, CVEDataset, CPEDataset\n",
     "from sec_certs.utils.pandas import (\n",
     "    compute_cve_correlations,\n",
@@ -82,15 +83,11 @@
     "\n",
     "# # Remote instantiation (takes approx. 10 minutes to complete)\n",
     "# dset: CCDataset = CCDataset.from_web_latest(path=\"dset\", auxiliary_datasets=True)\n",
+    "# dset.load_auxiliary_datasets()\n",
     "\n",
-    "# print(\"Downloading dataset of maintenance updates\")\n",
-    "# main_dset: CCDatasetMaintenanceUpdates = CCDatasetMaintenanceUpdates.from_web_latest()\n",
-    "\n",
-    "# print(\"Downloading CPE dataset\")\n",
-    "# cpe_dset: CPEDataset = dset.auxiliary_datasets.cpe_dset\n",
-    "\n",
-    "# print(\"Downloading CVE dataset\")\n",
-    "# cve_dset: CVEDataset = dset.auxiliary_datasets.cve_dset"
+    "# main_dset: CCDatasetMaintenanceUpdates = dset.aux_handlers[CCMaintenanceUpdateDatasetHandler].dset\n",
+    "# cpe_dset: CPEDataset = dset.aux_handlers[CPEDatasetHandler].dset\n",
+    "# cve_dset: CVEDataset = dset.aux_handlers[CVEDatasetHandler].dset"
    ]
   },
   {

notebooks/fips/vulnerabilities.ipynb

@@ -1,10 +1,12 @@
 {
  "cells": [
   {
-   "metadata": {},
    "cell_type": "markdown",
-   "source": "# Vulnerability analysis",
-   "id": "3a0981d008383c12"
+   "id": "3a0981d008383c12",
+   "metadata": {},
+   "source": [
+    "# Vulnerability analysis"
+   ]
   },
   {
    "cell_type": "code",
@@ -21,6 +23,7 @@
     "from sec_certs.dataset.fips import FIPSDataset\n",
     "from sec_certs.dataset.cpe import CPEDataset\n",
     "from sec_certs.dataset.cve import CVEDataset\n",
+    "from sec_certs.dataset.auxiliary_dataset_handling import CPEDatasetHandler, CVEDatasetHandler\n",
     "from sec_certs.utils.pandas import expand_df_with_cve_cols\n",
     "import pandas as pd\n",
     "import seaborn as sns\n",
@@ -47,8 +50,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cve_dset: CVEDataset = dset.auxiliary_datasets.cve_dset\n",
-    "cpe_dset: CPEDataset = dset.auxiliary_datasets.cpe_dset"
+    "dset.load_auxiliary_datasets()\n",
+    "cve_dset: CVEDataset = dset.aux_handlers[CVEDatasetHandler].dset\n",
+    "cpe_dset: CPEDataset = dset.aux_handlers[CPEDatasetHandler].dset"
    ]
   },
   {
@@ -181,14 +185,6 @@
     "g = sns.relplot(data=df_cve_rich, x=\"level\", y=\"avg_cve_score\")\n",
     "plt.show()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6c3c2ec4-3fab-48ad-aacb-6f54277abe66",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {

pyproject.toml

@@ -35,7 +35,7 @@
     "matplotlib",
     "numpy",
     "pandas",
-    "pdftotext",
+    "pdftotext>=3.0.0",
     "pikepdf",
     "Pillow>=9.2.0",
     "pypdf[crypto]>=3.1.0",

requirements/all_requirements.txt

@@ -20,8 +20,6 @@ appnope==0.1.3
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -138,10 +136,6 @@ docutils==0.19
     #   sphinx
 evaluate==0.4.1
     # via setfit
-exceptiongroup==1.2.2
-    # via
-    #   ipython
-    #   pytest
 executing==2.0.1
     # via stack-data
 fastjsonschema==2.19.0
@@ -430,7 +424,7 @@ parso==0.8.3
     # via jedi
 partd==1.4.1
     # via dask
-pdftotext==2.2.2
+pdftotext==3.0.0
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
@@ -722,15 +716,6 @@ tifffile==2023.9.26
     # via scikit-image
 tokenizers==0.15.0
     # via transformers
-tomli==2.0.1
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-    #   setuptools-scm
 toolz==0.12.0
     # via
     #   dask
@@ -786,7 +771,6 @@ types-requests==2.31.0.10
 typing-extensions==4.8.0
     # via
     #   alembic
-    #   cloudpathlib
     #   huggingface-hub
     #   mypy
     #   myst-nb

requirements/dev_requirements.txt

@@ -2,6 +2,7 @@ accessible-pygments==0.0.4
     # via pydata-sphinx-theme
 aiohappyeyeballs==2.4.0
     # via aiohttp
+aiohttp==3.10.11
 aiohttp==3.10.11
     # via
     #   datasets
@@ -12,14 +13,13 @@ alabaster==0.7.13
     # via sphinx
 annotated-types==0.6.0
     # via pydantic
+appnope==0.1.4
 appnope==0.1.4
     # via
     #   ipykernel
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==5.0.1
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -86,6 +86,8 @@ datasets==2.15.0
     # via sec-certs (./../pyproject.toml)
 dateparser==1.2.0
     # via sec-certs (./../pyproject.toml)
+dateparser==1.2.0
+    # via sec-certs (./../pyproject.toml)
 debugpy==1.8.0
     # via ipykernel
 decorator==5.1.1
@@ -105,10 +107,6 @@ docutils==0.19
     #   myst-parser
     #   pydata-sphinx-theme
     #   sphinx
-exceptiongroup==1.2.2
-    # via
-    #   ipython
-    #   pytest
 executing==2.0.1
     # via stack-data
 fastjsonschema==2.19.0
@@ -229,6 +227,7 @@ murmurhash==1.0.10
     #   preshed
     #   spacy
     #   thinc
+mypy==1.13.0
 mypy==1.13.0
     # via sec-certs (./../pyproject.toml)
 mypy-extensions==1.0.0
@@ -293,7 +292,7 @@ pandas==2.1.3
     #   tabula-py
 parso==0.8.3
     # via jedi
-pdftotext==2.2.2
+pdftotext==3.0.0
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
@@ -325,6 +324,8 @@ prompt-toolkit==3.0.41
     # via ipython
 propcache==0.2.0
     # via yarl
+propcache==0.2.0
+    # via yarl
 psutil==5.9.6
     # via
     #   ipykernel
@@ -388,6 +389,7 @@ pytest-profiling==1.7.0
 python-dateutil==2.8.2
     # via
     #   dateparser
+    #   dateparser
     #   jupyter-client
     #   matplotlib
     #   pandas
@@ -398,6 +400,9 @@ pytz==2023.3.post1
     # via
     #   dateparser
     #   pandas
+    # via
+    #   dateparser
+    #   pandas
 pyyaml==6.0.1
     # via
     #   datasets
@@ -419,6 +424,8 @@ referencing==0.31.0
     #   jsonschema-specifications
 regex==2024.9.11
     # via dateparser
+regex==2024.9.11
+    # via dateparser
 requests==2.32.0
     # via
     #   datasets
@@ -433,6 +440,7 @@ rpds-py==0.13.1
     # via
     #   jsonschema
     #   referencing
+ruff==0.7.4
 ruff==0.7.4
     # via sec-certs (./../pyproject.toml)
 scikit-learn==1.5.0
@@ -517,15 +525,6 @@ thinc==8.2.1
     # via spacy
 threadpoolctl==3.2.0
     # via scikit-learn
-tomli==2.1.0
-    # via
-    #   build
-    #   coverage
-    #   mypy
-    #   pip-tools
-    #   pyproject-hooks
-    #   pytest
-    #   setuptools-scm
 tornado==6.4.1
     # via
     #   ipykernel
@@ -559,7 +558,6 @@ types-requests==2.31.0.10
     # via sec-certs (./../pyproject.toml)
 typing-extensions==4.8.0
     # via
-    #   cloudpathlib
     #   huggingface-hub
     #   mypy
     #   myst-nb
@@ -573,6 +571,8 @@ tzdata==2023.3
     # via pandas
 tzlocal==5.2
     # via dateparser
+tzlocal==5.2
+    # via dateparser
 urllib3==2.2.2
     # via
     #   requests
@@ -600,6 +600,7 @@ wrapt==1.16.0
     # via deprecated
 xxhash==3.4.1
     # via datasets
+yarl==1.17.2
 yarl==1.17.2
     # via aiohttp
 zipp==3.19.1

requirements/nlp_requirements.txt

@@ -16,8 +16,6 @@ appnope==0.1.3
     #   ipython
 asttokens==2.4.1
     # via stack-data
-async-timeout==4.0.3
-    # via aiohttp
 attrs==23.1.0
     # via
     #   aiohttp
@@ -109,8 +107,6 @@ distro==1.8.0
     # via tabula-py
 evaluate==0.4.1
     # via setfit
-exceptiongroup==1.2.2
-    # via ipython
 executing==2.0.1
     # via stack-data
 filelock==3.13.1
@@ -343,7 +339,7 @@ parso==0.8.3
     # via jedi
 partd==1.4.1
     # via dask
-pdftotext==2.2.2
+pdftotext==3.0.0
     # via sec-certs (./../pyproject.toml)
 pexpect==4.8.0
     # via ipython
@@ -559,8 +555,6 @@ tifffile==2023.9.26
     # via scikit-image
 tokenizers==0.15.0
     # via transformers
-tomli==2.0.1
-    # via setuptools-scm
 toolz==0.12.0
     # via
     #   dask
@@ -608,7 +602,6 @@ typer==0.9.0
 typing-extensions==4.8.0
     # via
     #   alembic
-    #   cloudpathlib
     #   huggingface-hub
     #   panel
     #   pydantic