Add script to run analytics (#18)

* chkpt

* chkpt

* chkpt

* chkpt

* chkpt

* chkpt

* update lookups destination location

* move analytics to mitre_car

* one more reference to old table name

* fmt

* update path

* udpate requirements based on latest pipfile

* bump version

.env-default

@@ -1,4 +1,6 @@
 # Create your own .env file in the root of this project.
 # DATAPATH is required and should be the dir level above any data sets.
+# LOOKUPS is optional and should be the dir containing lookup/enrichement tables
 # Note: on windows you should still use forward slashes
 DATAPATH=/some/path/to/datasets
+LOOKUPS=/some/path/to/lookups

.github/workflows/main.yml

@@ -14,7 +14,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.10'
+          python-version: '3.11'
           architecture: x64
       - name:  Setup
         run: make venv

Makefile

@@ -19,7 +19,7 @@ test:
 ci: fmt-check lint test
 
 venv:
-	pip3 install --user pipenv
+	pip3 install pipenv
 	pipenv install --dev
 
 build:
@@ -34,6 +34,9 @@ source-install:
 
 setup: venv source-install cleanpynb
 
+requirements:
+	pipenv run pip freeze > requirements.txt
+
 cleanpynb:
 	pip install nbstripout
 	nbstripout --install --attributes .gitattributes

Pipfile

@@ -16,6 +16,7 @@ types-pyyaml = "*"
 altair = "*"
 boto3 = "*"
 duckdb = "*"
+fsspec = "*"
 humanfriendly = "*"
 importlib_resources = "*"
 ipyfilechooser = "*"
@@ -37,4 +38,4 @@ magic-duckdb = "*"
 pyyaml = "*"
 
 [requires]
-python_version = "3.10"
+python_version = "3.11"

requirements.txt

@@ -1,137 +1,153 @@
--i https://pypi.org/simple
-altair==5.0.1
+altair==5.1.2
 antlr4-python3-runtime==4.9.3
 appdirs==1.4.4
-asttokens==2.2.1
-attrs==23.1.0; python_version >= '3.7'
+appnope==0.1.3
+arrow==1.3.0
+asttokens==2.4.0
+attrs==23.1.0
 backcall==0.2.0
-backoff==2.2.1; python_version >= '3.7' and python_version < '4.0'
-boto3==1.28.11
-botocore==1.31.11; python_version >= '3.7'
-cattrs==23.1.2; python_version >= '3.7'
-certifi==2023.7.22; python_version >= '3.6'
-charset-normalizer==3.2.0; python_full_version >= '3.7.0'
-click==8.1.6; python_version >= '3.7'
-colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
+backoff==2.2.1
+black==23.7.0
+boto3==1.28.62
+botocore==1.31.62
+cattrs==23.1.2
+certifi==2023.7.22
+chardet==5.2.0
+charset-normalizer==3.3.0
+click==8.1.7
+colorama==0.4.6
 colour==0.1.5
-comm==0.1.3; python_version >= '3.6'
-contourpy==1.1.0; python_version >= '3.8'
+comm==0.1.4
+contourpy==1.1.1
 cpe==1.2.1
 cybox==2.1.0.21
-cycler==0.11.0; python_version >= '3.6'
-debugpy==1.6.7; python_version >= '3.7'
-decorator==5.1.1; python_version >= '3.5'
-deepdiff==6.3.1; python_version >= '3.7'
-drawsvg==2.2.0
-duckdb==0.8.1
+cycler==0.12.1
+decorator==5.1.1
+deepdiff==6.6.0
+diff-cover==7.7.0
+drawsvg==2.3.0
+duckdb==0.9.0
 duckdb-engine==0.9.2
-et-xmlfile==1.1.0; python_version >= '3.6'
-exceptiongroup==1.1.2; python_version < '3.11'
-executing==1.2.0
-fonttools==4.41.1; python_version >= '3.8'
-gitdb==4.0.10; python_version >= '3.7'
-gitpython==3.1.32
-greenlet==2.0.2; platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+et-xmlfile==1.1.0
+executing==2.0.0
+fastjsonschema==2.18.0
+fonttools==4.43.1
+fqdn==1.5.1
+fsspec==2023.9.2
+gitdb==4.0.10
+GitPython==3.1.37
 humanfriendly==10.0
-idna==3.4; python_version >= '3.5'
-importlib-resources==6.0.0
-iniconfig==2.0.0; python_version >= '3.7'
+idna==3.4
+importlib-resources==6.1.0
+iniconfig==2.0.0
 ipyfilechooser==0.6.0
-ipykernel==6.25.0; python_version >= '3.8'
-ipython==8.14.0; python_version >= '3.9'
+ipython==8.16.1
 ipython-genutils==0.2.0
-ipywidgets==8.0.7; python_version >= '3.7'
-jedi==0.18.2; python_version >= '3.6'
-jinja2==3.0.3
+ipywidgets==8.1.1
+isoduration==20.11.0
+isort==5.12.0
+jedi==0.19.1
+Jinja2==3.0.3
 jinjasql==0.1.8
-jmespath==1.0.1; python_version >= '3.7'
-jsonschema==4.18.4; python_version >= '3.8'
-jsonschema-specifications==2023.7.1; python_version >= '3.8'
-jupysql==0.8.0
-jupyter-client==8.3.0; python_version >= '3.8'
-jupyter-core==5.3.1; python_version >= '3.8'
-jupyterlab-widgets==3.0.8; python_version >= '3.7'
-kiwisolver==1.4.4; python_version >= '3.7'
-loguru==0.7.0; python_version >= '3.5'
-lxml==4.9.3; python_version == '2.7' and python_version >= '3.5'
+jmespath==1.0.1
+jsonpointer==2.4
+jsonschema==4.17.3
+jsonschema-specifications==2023.7.1
+jupysql==0.10.2
+jupysql-plugin==0.2.5
+jupyter_core==5.3.1
+jupyterlab-widgets==3.0.9
+kiwisolver==1.4.5
+loguru==0.7.2
+lxml==4.9.3
 maec==4.1.0.17
 magic-duckdb==0.1.26
-markdown==3.4.4; python_version >= '3.7'
-markdown-it-py==3.0.0; python_version >= '3.8'
-markupsafe==2.1.3; python_version >= '3.7'
-matplotlib==3.7.2
-matplotlib-inline==0.1.6; python_version >= '3.5'
-mdurl==0.1.2; python_version >= '3.7'
+Markdown==3.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.0
+matplotlib-inline==0.1.6
+mdurl==0.1.2
 mitreattack-python==2.0.14
 mixbox==1.0.5
 monotonic==1.6
-nest-asyncio==1.5.6; python_version >= '3.5'
-netaddr==0.8.0
+mypy==1.5.1
+mypy-extensions==1.0.0
+nbformat==5.9.2
+nbstripout==0.6.1
+netaddr==0.9.0
 networkx==3.1
-numpy==1.25.1; python_version >= '3.9'
-openpyxl==3.1.2; python_version >= '3.6'
-ordered-set==4.1.0; python_version >= '3.7'
-packaging==23.1; python_version >= '3.7'
-pandas==2.0.3
-parso==0.8.3; python_version >= '3.6'
-pexpect==4.8.0; sys_platform != 'win32'
+numpy==1.26.0
+openpyxl==3.1.2
+ordered-set==4.1.0
+packaging==23.2
+pandas==2.1.1
+parso==0.8.3
+pathspec==0.11.2
+pexpect==4.8.0
 pickleshare==0.7.5
-pillow==10.0.0; python_version >= '3.8'
-platformdirs==3.9.1; python_version >= '3.7'
-ploomber-core==0.2.13
-pluggy==1.2.0; python_version >= '3.7'
-pluralizer==1.2.0; python_version >= '3.6'
-pooch==1.7.0; python_version >= '3.7'
-posthog==3.0.1
-prettytable==3.8.0; python_version >= '3.8'
-prompt-toolkit==3.0.39; python_full_version >= '3.7.0'
-psutil==5.9.5; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+Pillow==10.0.1
+platformdirs==3.11.0
+ploomber-core==0.2.14
+pluggy==1.3.0
+pluralizer==1.2.0
+pooch==1.7.0
+posthog==3.0.2
+prettytable==3.9.0
+prompt-toolkit==3.0.39
 ptyprocess==0.7.0
 pure-eval==0.2.2
-pyarrow==12.0.1
-pycountry==22.3.5; python_version >= '3.6' and python_version < '4'
-pygments==2.15.1; python_version >= '3.7'
-pyparsing==3.0.9; python_full_version >= '3.6.8'
+pyarrow==13.0.0
+pycountry==22.3.5
+Pygments==2.16.1
+pyparsing==3.1.1
+pyrsistent==0.19.3
 pytest==7.4.0
-python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+python-dateutil==2.8.2
 python-dotenv==1.0.0
-pytz==2023.3
-pyyaml==6.0.1; python_version >= '3.6'
-pyzmq==25.1.0; python_version >= '3.6'
-referencing==0.30.0; python_version >= '3.8'
-requests==2.31.0; python_version >= '3.7'
-requests-cache==1.1.0; python_version >= '3.7' and python_version < '4.0'
-rich==13.4.2; python_full_version >= '3.7.0'
-rpds-py==0.9.2; python_version >= '3.8'
-s3transfer==0.6.1; python_version >= '3.7'
-setuptools==68.0.0; python_version >= '3.7'
-simplejson==3.19.1; python_version >= '2.5' and python_version not in '3.0, 3.1, 3.2, 3.3'
-six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-smmap==5.0.0; python_version >= '3.6'
-sqlalchemy==2.0.19; python_version >= '3.7'
-sqlglot==17.8.0
-sqlparse==0.4.4; python_version >= '3.5'
-stack-data==0.6.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.30.2
+regex==2023.8.8
+requests==2.31.0
+requests-cache==1.1.0
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rich==13.6.0
+rpds-py==0.10.0
+s3transfer==0.7.0
+simplejson==3.19.2
+six==1.16.0
+smmap==5.0.1
+SQLAlchemy==2.0.21
+sqlfluff==2.3.0
+sqlglot==18.11.6
+sqlparse==0.4.4
+stack-data==0.6.3
 stix==1.2.0.11
-stix2==3.0.1; python_version >= '3.6'
-stix2-elevator==4.1.7; python_version >= '3.7'
-stix2-patterns==2.0.0; python_version >= '3.6'
-stix2-validator==3.0.2
+stix2==3.0.1
+stix2-elevator==4.1.7
+stix2-patterns==2.0.0
+stix2-validator==3.1.4
 stixmarx==1.0.8
-tabulate==0.9.0; python_version >= '3.7'
+tabulate==0.9.0
 taxii2-client==2.3.0
+tblib==2.0.0
+tokenize-rt==5.2.0
 toml==0.10.2
-tomli==2.0.1; python_version < '3.11'
-toolz==0.12.0; python_version >= '3.5'
-tornado==6.3.2; python_version >= '3.8'
-tqdm==4.65.0
-traitlets==5.9.0; python_version >= '3.7'
-typer==0.9.0; python_version >= '3.6'
-typing-extensions==4.7.1; python_version < '3.11'
-tzdata==2023.3; python_version >= '2'
-url-normalize==1.4.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-urllib3==1.26.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-wcwidth==0.2.6
+toolz==0.12.0
+tqdm==4.66.1
+traitlets==5.11.2
+typer==0.9.0
+types-python-dateutil==2.8.19.14
+types-PyYAML==6.0.12.11
+typing_extensions==4.8.0
+tzdata==2023.3
+uri-template==1.3.0
+url-normalize==1.4.3
+urllib3==2.0.6
+wcwidth==0.2.8
 weakrefmethod==1.0.3
-widgetsnbextension==4.0.8; python_version >= '3.7'
-xlsxwriter==3.1.2; python_version >= '3.6'
+webcolors==1.13
+widgetsnbextension==4.0.9
+XlsxWriter==3.1.7

setup.cfg

@@ -26,6 +26,7 @@ exclude =
 console_scripts = 
     collectbinaries = wintappy.etlutils.collectbinaries:main
     download_from_s3 = wintappy.etlutils.download_from_s3:main
+    run_enrichment = wintappy.etlutils.run_enrichment:main
     rawtorolling = wintappy.etlutils.rawtorolling:main
     rawtostdview = wintappy.etlutils.rawtostdview:main
 

setup.py

@@ -4,6 +4,7 @@
     "altair",
     "boto3",
     "duckdb",
+    "fsspec",
     "gitpython",
     "humanfriendly",
     "importlib_resources",

tests/database/test_wintap_duckdb.py

@@ -36,22 +36,43 @@ def test_write(self, connection: mock.MagicMock) -> None:
 
     @mock.patch("datetime.datetime")
     @mock.patch("duckdb.DuckDBPyConnection")
-    def test_insert_analytics(
+    def test_insert_analytics_results(
         self, connection: mock.MagicMock, mock_datetime: mock.MagicMock
     ) -> None:
         wintap_db = WintapDuckDB(WintapDuckDBOptions(connection, self.dataset_path))
         mock_time = "1689378948"
         mock_datetime.strftime.return_value = mock_time
         mock_analytic_id = "my-cool-analytic"
         mock_entity_type = "not_the-pid-hash"
-        expected_sql = f"INSERT INTO analytics_results(\n    entity,\n    analytic_id,\n    time,\n    entity_type\n)\nVALUES (\n    '{mock_entity_type}',\n    '{mock_analytic_id}',\n    to_timestamp({int(mock_time)}),\n    'pid_hash'\n)"
-        wintap_db.insert_analytics_table(
+        expected_sql = f"INSERT INTO mitre_labels(\n    entity,\n    analytic_id,\n    time,\n    entity_type\n)\nVALUES (\n    '{mock_entity_type}',\n    '{mock_analytic_id}',\n    to_timestamp({int(mock_time)}),\n    'pid_hash'\n)"
+        wintap_db.insert_analytics_results_table(
             mock_analytic_id,
             mock_entity_type,
             event_time=mock_datetime,
         )
         connection.execute.assert_called_with(expected_sql)
 
+    @mock.patch("datetime.datetime")
+    @mock.patch("duckdb.DuckDBPyConnection")
+    def test_insert_analytics(
+        self, connection: mock.MagicMock, mock_datetime: mock.MagicMock
+    ) -> None:
+        wintap_db = WintapDuckDB(WintapDuckDBOptions(connection, self.dataset_path))
+        mock_analytic_id = "my-cool-analytic"
+        mock_technique_id = "tech-id"
+        mock_stix_type = "tech-id-stix"
+        mock_tactic_id = "tactic-id"
+        mock_tactic_stix_type = "tactic-id-stix"
+        expected_sql = f"INSERT INTO mitre_car(\n    analytic_id,\n    technique_id,\n    technique_stix_type,\n    tactic_id,\n    tactic_stix_type\n)\nVALUES (\n    '{mock_analytic_id}',\n    '{mock_technique_id}',\n    '{mock_stix_type}',\n    '{mock_tactic_id}',\n    '{mock_tactic_stix_type}'\n)"
+        wintap_db.insert_analytics_table(
+            mock_analytic_id,
+            mock_technique_id,
+            mock_stix_type,
+            mock_tactic_id,
+            mock_tactic_stix_type,
+        )
+        connection.execute.assert_called_with(expected_sql)
+
     @mock.patch("duckdb.DuckDBPyConnection")
     def test_is_not_table_or_view(self, connection: mock.MagicMock) -> None:
         wintap_db = WintapDuckDB(WintapDuckDBOptions(connection, self.dataset_path))

wintappy/__init__.py

@@ -1 +1 @@
-VERSION = "0.1.1"
+VERSION = "0.1.2"

wintappy/analytics/constants.py

@@ -25,3 +25,5 @@
 ## MITRE ATT&CK Constants
 ATTACK_STIX_REPO_URL = "https://github.com/mitre-attack/attack-stix-data"
 LATEST_ENTERPRISE_DEFINITION = "enterprise-attack/enterprise-attack.json"
+TECHNIQUE_STIX_TYPE = "attack-pattern"
+TACTIC_STIX_TYPE = "x-mitre-tactic"