Update Customizations and Bundling Creation (#31)

* Update customizations.py
* Finish the Customizations work
* Fix bug with cmd exec cwd
* Update examples with new pack structure

Author: GeekMasher

codeqlsummarize/exporters/customizations.py

@@ -5,6 +5,7 @@
 
 from codeqlsummarize.models import CodeQLDatabase, GitHub
 from codeqlsummarize.generator import QUERIES
+from codeqlsummarize.utils import findCodeQLCli
 
 logger = logging.getLogger("codeqlsummarize.exporters")
 
@@ -19,7 +20,7 @@
 """
 
 CODEQL_CUSTOMIZATION = """\
-private class {name}{type}Custom extends {models} {{
+private class {name}{type}Custom extends {models}Csv {{
   override predicate row(string row) {{
     row = [
 {rows}
@@ -33,7 +34,7 @@ def saveQLL(
     database: CodeQLDatabase, output_customizations: str, github: GitHub, **kargs
 ):
     padding = " " * 6
-    owner = github.owner.replace("-", "_")
+    owner = github.owner.replace("-", "_").lower()
 
     models = {}
     # initially populate data
@@ -47,7 +48,7 @@ def saveQLL(
         if len(summary.rows) == 0:
             models[sname] = f"// No {sname} found\n"
             continue
-        for mad in summary.rows:
+        for mad in sorted(summary.rows):
             rows += f'{padding}"{mad}"'
 
             if len(summary.rows) > counter:
@@ -118,35 +119,37 @@ def exportCustomizations(
 def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs):
     logger.debug(f"Output directory :: {output}")
 
-    owner = github.owner.replace("-", "_")
+    owner = github.owner.replace("-", "_").lower()
 
     if not github or not github.owner:
         raise Exception("Failed to export Bundle: No owner / repo name set")
 
+    codeql_pack_path = f"{database.language}-summarize"
+    codeql_pack_name = f"{owner}/{codeql_pack_path}"
+
     # Create root for language
-    root = os.path.join(output, database.language, owner)
-    os.makedirs(root, exist_ok=True)
-    logger.debug(f"Root for language :: {root}")
-
-    # Create language files
-    codeql_lang_lock = os.path.join(root, "codeql-pack.lock.yml")
-    if not os.path.exists(codeql_lang_lock):
-        logger.debug(f"Creating Language Lock file :: {codeql_lang_lock}")
-        with open(codeql_lang_lock, "w") as handle:
-            handle.write(CODEQL_LOCK.format(language=database.language))
-
-    codeql_lang_pack = os.path.join(root, "qlpack.yml")
-    if not os.path.exists(codeql_lang_pack):
-        logger.debug(f"Creating Language Pack file :: {codeql_lang_pack}")
-        with open(codeql_lang_pack, "w") as handle:
-            handle.write(
-                CODEQL_PACK.format(
-                    owner=owner, version="0.1.0", language=database.language
-                )
-            )
+    root = os.path.join(output, codeql_pack_path)
+
+    codeql = findCodeQLCli()
+
+    if not os.path.exists(root) and codeql:
+        logger.info("Generating CodeQL Summarize Pack")
+        codeql("pack", "init", "--version=0.0.1", "--extractor", database.language, codeql_pack_path, cwd=output)
+
+    if not os.path.exists(os.path.join(root, "qlpack.yml")):
+        raise Exception("Pack wasn't found")
+
+    # Create README
+    readme = os.path.join(root, "README.md")
+    if not os.path.exists(readme):
+        with open(readme, "w") as handle:
+            handle.write("# CodeQL Summarize Pack\n")
+
+    logger.debug(f"Root Pack Path :: {root}")
 
     # Create language subfolder (if needed)
-    sub = os.path.join(root, owner, database.language)
+    sub = os.path.join(root, owner, codeql_pack_path.replace("-", "_"))
+    logger.debug(f"Checking sub pack path exists: {sub}")
     os.makedirs(sub, exist_ok=True)
 
     name = database.display_name(owner=owner) + "Generated"
@@ -157,13 +160,20 @@ def exportBundle(database: CodeQLDatabase, output: str, github: GitHub, **kargs)
     # Dynamically update Customizations.qll
     customizations_path = os.path.join(sub, "Customizations.qll")
     customizations_data = ""
-    for custom in os.listdir(sub):
+    
+    codeql_files = os.listdir(sub)
+    if not codeql_files:
+        logger.error(f"This is a major issue and please report in the GitHub issues")
+        raise Exception("Something is really wrong here...")
+
+    for custom in codeql_files:
         if custom == "Customizations.qll":
             continue
 
         custom = custom.replace(".qll", "")
 
-        impt = f"    private import {owner}.{database.language}.{custom}\n"
+        impt = f"    private import {owner}.{database.language}_summarize.{custom}\n"
+
         customizations_data += impt
 
     with open(customizations_path, "w") as handle:

codeqlsummarize/models.py

@@ -16,7 +16,7 @@
 
 @dataclass
 class Summaries:
-    rows: List[str]
+    rows: List[str] = field(default_factory=list)
 
 
 @dataclass

examples/java-summarize/README.md

@@ -0,0 +1 @@
+# CodeQL Summarize Pack

examples/java-summarize/geekmasherorg/java_summarize/Customizations.qll

@@ -0,0 +1,8 @@
+// This file is Automatically Generated based on the files in-side this relative
+// directory. This makes it easier to automate this process.
+import java
+
+module geekmasherorg {
+    private import geekmasherorg.java_summarize.EsapiEsapiJavaLegacyGenerated
+
+}

…y/java/EsapiEsapiJavaLegacyGenerated.qll …marize/EsapiEsapiJavaLegacyGenerated.qllexamples/java/advanced_security/advanced_security/java/EsapiEsapiJavaLegacyGenerated.qll renamed to examples/java-summarize/geekmasherorg/java_summarize/EsapiEsapiJavaLegacyGenerated.qll examples/java/advanced_security/advanced_security/java/EsapiEsapiJavaLegacyGenerated.qll renamed to examples/java-summarize/geekmasherorg/java_summarize/EsapiEsapiJavaLegacyGenerated.qll

@@ -1,7 +1,7 @@
 import java
 private import semmle.code.java.dataflow.ExternalFlow
 
-private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModel {
+private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModelCsv {
   override predicate row(string row) {
     row = [
       "org.owasp.esapi.codecs;Base64;true;decodeFileToFile;(String,String);;Argument[1];create-file;generated",
@@ -27,7 +27,7 @@ private class EsapiEsapiJavaLegacySinkModelCustom extends SinkModel {
   }
 }
 
-private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModel {
+private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModelCsv {
   override predicate row(string row) {
     row = [
       "org.owasp.esapi.filters;SecurityWrapperRequest;true;getParameter;(String,boolean);;ReturnValue;remote;generated",
@@ -43,7 +43,7 @@ private class EsapiEsapiJavaLegacySourceModelCustom extends SourceModel {
   }
 }
 
-private class EsapiEsapiJavaLegacySummaryModelCustom extends SummaryModel {
+private class EsapiEsapiJavaLegacySummaryModelCustom extends SummaryModelCsv {
   override predicate row(string row) {
     row = [
       "java.util;Properties;true;getProperty;(String);;Argument[-1];ReturnValue;taint;generated",

examples/java-summarize/qlpack.yml

@@ -0,0 +1,5 @@
+---
+library: false
+name: java-summarize
+version: 0.0.1
+extractor: java