v0.2.0

- Add Data Extensions support

Author: GeekMasher

Pipfile

@@ -8,9 +8,10 @@ black = "*"
 pyright = "*"
 
 [requires]
-python_version = "3.9"
+python_version = "3.10"
 
 [scripts]
+main = "python3 codeqlsummurize/__main__.py"
 lint = "black ."
 format = "black ."
 tests = "python -m unittest discover -v -s ./tests -p test_*.py"

codeqlsummarize/__main__.py

@@ -24,8 +24,8 @@
 parser.add_argument(
     "-f",
     "--format",
-    default="bundle",
-    help="Export format (`json`, `customizations`, `mad`, `bundle`)",
+    default="extensions",
+    help="Export format (`json`, `customizations`, `extensions`, `bundle`)",
 )
 parser.add_argument("-i", "--input", help="Input / Project File")
 parser.add_argument(

codeqlsummarize/__version__.py

@@ -1,7 +1,7 @@
 
 __title__ = "CodeQL Summarize"
 __name__ = "codeqlsummarize"
-__version__ = "0.1.0"
+__version__ = "0.2.0"
 
 __description__ = "GitHub CodeQL Summaries Toolkit"
 __summary__ = """\
@@ -11,7 +11,7 @@
 __url__ = "https://github.com/advanced-security/gh-codeql-summarize"
 
 __license__ = "MIT License"
-__copyright__ = "Copyright (c) 2022, GitHub"
+__copyright__ = "Copyright (c) 2023, GitHub"
 
 __author__ = "GitHub Field Team"
 __email__ = ""

codeqlsummarize/exporters/__init__.py

@@ -1,8 +1,10 @@
 from codeqlsummarize.exporters.exptjson import exportToJson
 from codeqlsummarize.exporters.customizations import exportBundle, exportCustomizations
+from codeqlsummarize.exporters.extensions import exportDataExtensions
 
 EXPORTERS = {
     "json": exportToJson,
+    "extensions": exportDataExtensions,
     "customizations": exportCustomizations,
     "bundle": exportBundle,
 }

codeqlsummarize/exporters/extensions.py

@@ -0,0 +1,78 @@
+import os
+import yaml
+import logging
+
+from codeqlsummarize.models import CodeQLDatabase, GitHub
+
+logger = logging.getLogger("codeqlsummarize.exporters.extensions")
+
+CODEQL_EXTENSION = """\
+  - addsTo:
+      pack: codeql/{language}-queries
+      extensible: {extensible}
+    data:
+{rows}
+"""
+
+EXTENSIBLE = {
+    "SinkModel": "sinkModel",
+    "SourceModel": "sourceModel",
+    "SummaryModel": "summaryModel",
+}
+
+
+def exportDataExtensions(database: CodeQLDatabase, output: str, github: GitHub, **kargs):
+    logger.info("Running export to Data Extensions")
+
+    if database.language == "javascript":
+        logger.warning("Skipping JavaScript for now")
+        return
+
+    # Get the CodeQL pack for the language
+    codeqlPack = findCodeQLPack(output, database.language)
+    os.makedirs(os.path.join(codeqlPack, "generated"), exist_ok=True)
+
+    if github.owner:
+        os.makedirs(os.path.join(codeqlPack, "generated", github.owner), exist_ok=True)
+        extensions_file = os.path.join(codeqlPack, "generated", github.owner, f"{database.name}.yml")
+    else:
+        extensions_file = os.path.join(codeqlPack, "generated", f"{database.name}.yml")
+
+    data = "extensions:\n"
+    for sname, summary in database.summaries.items():
+        if len(summary.rows) == 0:
+            continue
+
+        summary_rows = ""
+        for mad in sorted(summary.rows):
+            m = mad.split(";")
+            summary_rows += "      - "
+            summary_rows += f'["{m[0]}", "{m[1]}", {m[2]}, "{m[3]}", "{m[4]}", "{m[5]}", "{m[6]}", "{m[7]}", "{m[8]}"]\n'
+
+        data += CODEQL_EXTENSION.format(
+            rows=summary_rows,
+            language=database.language,
+            extensible=EXTENSIBLE.get(sname, "sinkModel")
+        )
+
+    logger.info(f"Writing Data Extensions to: {extensions_file}")
+    with open(extensions_file, "w") as handle:
+        handle.write(data)
+
+
+def findCodeQLPack(location: str, language: str) -> str:
+    """Find the CodeQL pack for the given language in the output directory"""
+
+    if os.path.isfile(location):
+        raise Exception(f"Directory {location} does not exist")
+
+    for root, dirs, files in os.walk(location):
+        for file in files:
+            if file == "qlpack.yml":
+                with open(os.path.join(root, file), "r") as f:
+                    qlpack = yaml.safe_load(f)
+                    
+                    if f"codeql/{language}-queries" in qlpack.get("extensionTargets", []):
+                        return root
+
+    raise Exception(f"Could not find CodeQL pack for {language} in {location}")

codeqlsummarize/generator.py

@@ -23,7 +23,7 @@
 QUERIES = {
     "SinkModel": "CaptureSinkModels.ql",
     "SourceModel": "CaptureSourceModels.ql",
-    "SummaryModel": "CaptureSummaryModels.ql",
+    # "SummaryModel": "CaptureSummaryModels.ql",
 }
 
 
@@ -47,7 +47,7 @@ def getModelGeneratorQuery(self, name) -> Optional[str]:
         query_file = QUERIES.get(name)
 
         if query_file:
-            return f"{self.pack_name}:utils/model-generator/{query_file}"
+            return f"{self.pack_name}:utils/modelgenerator/{query_file}"
 
         # Find in this repo
         return None

codeqlsummarize/models.py

@@ -21,7 +21,7 @@ class Summaries:
 
 @dataclass
 class GitHub:
-    owner: str = "Security"
+    owner: str = "security"
     repo: str = "codeql"
 
     endpoint: ClassVar[str] = "https://api.github.com"