Switch from dataclass-wizard to mashumaro (#389)

* switch from dataclass-wizard to mashumaro

Signed-off-by: Alex Goodman <[email protected]>

* split out static analysis from the test matrix

Signed-off-by: Alex Goodman <[email protected]>

* chore: bump poetry to v1.7.0

Signed-off-by: Weston Steimel <[email protected]>

* explicitly pass python version to job matrix

Signed-off-by: Alex Goodman <[email protected]>

* bust CI cache

Signed-off-by: Alex Goodman <[email protected]>

* use pip not pipx for poetry install

Signed-off-by: Alex Goodman <[email protected]>

* remove python 3.12 from the test matrix

Signed-off-by: Alex Goodman <[email protected]>

* paired matrix vars

Signed-off-by: Alex Goodman <[email protected]>

* add single test gate

Signed-off-by: Alex Goodman <[email protected]>

---------

Signed-off-by: Alex Goodman <[email protected]>
Signed-off-by: Weston Steimel <[email protected]>
Co-authored-by: Weston Steimel <[email protected]>

Author: wagoodman

.github/actions/bootstrap/action.yaml

@@ -4,15 +4,17 @@ inputs:
   go-version:
     description: "Go version to install"
     required: true
-    default: "1.19.x"
+    default: "1.21.x"
   python-version:
     description: "Python version to install"
     required: true
-    default: "3.9"
+    # note: the caller really drives this in a matrix strategy run for unit tests.
+    # this default value is used for the rest of the workflow.
+    default: "3.11"
   poetry-version:
     description: "Poetry version to install"
     required: true
-    default: "1.3.2"
+    default: "1.7.0"
   use-poetry-cache:
     description: "Restore poetry cache"
     required: true
@@ -28,7 +30,7 @@ inputs:
   cache-key-prefix:
     description: "Prefix all cache keys with this value"
     required: true
-    default: "12a830581c"
+    default: "8381c12a05"
   bootstrap-apt-packages:
     description: "Space delimited list of tools to install via apt"
     default: ""
@@ -49,7 +51,7 @@ runs:
     - name: Install poetry
       shell: bash
       run: |
-        pipx install poetry==${{ inputs.poetry-version }}
+        python -m pip install poetry==${{ inputs.poetry-version }}
         poetry self add "poetry-dynamic-versioning[plugin]"
 
     - name: Cache Poetry virtualenv
@@ -90,3 +92,10 @@ runs:
     - name: Install dependencies and package
       shell: bash
       run: poetry install
+
+    - name: Restore tox cache
+      id: cache-tox
+      uses: actions/cache@v3
+      with:
+        path: .tox
+        key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('tox.ini') }}

.github/workflows/release.yaml

@@ -28,13 +28,22 @@ jobs:
           git fetch origin main
           git merge-base --is-ancestor ${GITHUB_REF##*/} origin/main && echo "${GITHUB_REF##*/} is a commit on main!"
 
-      - name: Check validation results
+      - name: Check static analysis results
         uses: fountainhead/action-wait-for-check@297be350cf8393728ea4d4b39435c7d7ae167c93 #v1.1.1
-        id: validations
+        id: static-analysis
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           # This check name is defined as the github action job name (in .github/workflows/validations.yaml)
-          checkName: "Validations"
+          checkName: "Static Analysis"
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - name: Check test results
+        uses: fountainhead/action-wait-for-check@297be350cf8393728ea4d4b39435c7d7ae167c93 #v1.1.1
+        id: test
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          # This check name is defined as the github action job name (in .github/workflows/validations.yaml)
+          checkName: "Test Gate"
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
 
       - name: Check nightly quality gate results
@@ -50,9 +59,10 @@ jobs:
           intervalSeconds: 3
 
       - name: Release quality gate
-        if: steps.validations.conclusion != 'success' || steps.nightly-quality-gate.conclusion != 'success'
+        if: steps.static-analysis.conclusion != 'success' || steps.test.conclusion != 'success' || steps.nightly-quality-gate.conclusion != 'success'
         run: |
-          echo "Validations Status: ${{ steps.validations.conclusion }}"
+          echo "Static Analysis Status: ${{ steps.static-analysis.conclusion }}"
+          echo "Test Status: ${{ steps.test.conclusion }}"
           echo "Nightly Quality Gate Status: ${{ steps.nightly-quality-gate.conclusion }}"
           false
 

.github/workflows/validations.yaml

@@ -11,7 +11,8 @@ on:
 jobs:
 
   # note: the name for this check is referenced in release.yaml, do not change here without changing there
-  Validations:
+  Static-Analysis:
+    name: "Static Analysis"
     runs-on: ubuntu-22.04
     permissions:
       contents: read
@@ -27,19 +28,63 @@ jobs:
       - name: Run static analysis
         run: poetry run make static-analysis
 
-      - name: Run unit tests
-        run: poetry run make unit
-
       - name: Ensure quality gate tools are properly configured
         run: |
           cd tests/quality && make validate-test-tool-versions
 
+  Test:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        # note: this is not a single source of truth (this is also in the tox.ini)
+        python:
+          - version: '3.9'
+            toxEnv: py39
+
+          - version: '3.10'
+            toxEnv: py310
+
+          - version: '3.11'
+            toxEnv: py311
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
+        with:
+          # in order to properly resolve the version from git
+          fetch-depth: 0
+
+      - name: Bootstrap environment
+        uses: ./.github/actions/bootstrap
+        with:
+          python-version: ${{ matrix.python.version }}
+
+      - name: Run unit tests
+        run: poetry run tox -e ${{ matrix.python.toxEnv }}
+
       - name: Build assets
         run: poetry run make build
 
+  # this is to help facilitate ensuring all checks have run with the checks API for release
+  # see https://github.com/orgs/community/discussions/26822#discussioncomment-3305794
+  # as well as the release.yaml workflow
+  Test-Gate:
+    if: ${{ always() }}
+    runs-on: ubuntu-22.04
+    name: Test Gate
+    needs: [test]
+    steps:
+      - run: |
+          result="${{ needs.Test.result }}"
+          if [[ $result == "success" || $result == "skipped" ]]; then
+            exit 0
+          else
+            exit 1
+          fi
+
   Publish-PreProd:
     runs-on: ubuntu-22.04
-    needs: [Validations]
+    needs: [Static-Analysis, Test]
     if: github.ref == 'refs/heads/main'
     permissions:
       contents: read

Makefile

@@ -123,6 +123,10 @@ check-types: virtual-env-check  ## Run type checks (mypy)
 unit: virtual-env-check  ## Run unit tests
 	pytest --cov-report html --cov vunnel -v tests/unit/
 
+.PHONY: unit-matrix
+unit-matrix: virtual-env-check  ## Run unit tests for all supported python versions
+	tox
+
 
 ## Build-related targets #################################
 

poetry.lock

@@ -51,14 +51,14 @@ xxhash = "^3.1.0"
 cvss = "^2.6"
 python-dateutil = "^2.8.2"
 defusedxml = "^0.7.1"
-dataclass-wizard = "^0.22.2"
 orjson = "^3.8.6"
 SQLAlchemy = ">= 1.4.46, < 2.0"  # note: 1.4.x currently required for enterprise
 mergedeep = "^1.3.4"
 future = "^0.18.3"
 importlib-metadata = "^6.1.0"
 xsdata = {extras = ["cli", "lxml", "soap"], version = ">=22.12,<24.0"}
 pytest-snapshot = "^0.9.0"
+mashumaro = "^3.10"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.2"
@@ -80,6 +80,7 @@ dunamai = "^1.15.0"
 ruff = ">=0.0.254,<0.1.5"
 yardstick = {git = "https://github.com/anchore/yardstick", rev = "v0.9.1"}
 tabulate = "0.9.0"
+tox = "^4.11.3"
 
 [build-system]
 requires = ["poetry-core>=1.3.0", "poetry-dynamic-versioning"]

pyproject.toml

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -vv --color=yes

pytest.ini

@@ -6,7 +6,7 @@
 
 import mergedeep
 import yaml
-from dataclass_wizard import asdict, fromdict
+from mashumaro.mixins.dict import DataClassDictMixin
 
 from vunnel import providers
 
@@ -49,7 +49,7 @@ def __post_init__(self) -> None:
 
 
 @dataclass
-class Application:
+class Application(DataClassDictMixin):
     root: str = "./data"
     log: Log = field(default_factory=Log)
     providers: Providers = field(default_factory=Providers)
@@ -60,18 +60,15 @@ def load(path: str = ".vunnel.yaml") -> Application:
         with open(path, encoding="utf-8") as f:
             app_object = yaml.safe_load(f.read()) or {}
             # we need a full default application config first then merge the loaded config on top.
-            # Why? dataclass_wizard.fromdict() will create instances from the dataclass default
+            # Why? cls.from_dict() will create instances from the dataclass default
             # and NOT the field definition from the container. So it is possible to specify a
             # single field in the config and all other fields would be set to the default value
             # based on the dataclass definition and not any field(default_factory=...) hints
             # from the containing class.
-            instance = asdict(Application())
+            instance = Application().to_dict()
 
             mergedeep.merge(instance, app_object)
-            cfg = fromdict(
-                Application,
-                instance,
-            )
+            cfg = Application.from_dict(instance)
             if cfg is None:
                 raise FileNotFoundError("parsed empty config")
     except FileNotFoundError:

src/vunnel/cli/config.py

@@ -14,7 +14,7 @@
 
 @dataclass
 class Config:
-    security_advisories: dict[Any, str] = field(default_factory=lambda: amazon_security_advisories)
+    security_advisories: dict[Any, str] = field(default_factory=lambda: amazon_security_advisories.copy())
     runtime: provider.RuntimeConfig = field(
         default_factory=lambda: provider.RuntimeConfig(
             result_store=result.StoreStrategy.SQLITE,