Added helper function for converting gfm output

Author: dacoburn

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "socketsecurity"
-version = "2.1.3"
+version = "2.1.4"
 requires-python = ">= 3.10"
 license = {"file" = "LICENSE"}
 dependencies = [

socketsecurity/__init__.py

@@ -1,2 +1,2 @@
 __author__ = 'socket.dev'
-__version__ = '2.1.3'
+__version__ = '2.1.4'

socketsecurity/core/__init__.py

@@ -425,7 +425,7 @@ def create_packages_dict(self, sbom_artifacts: list[SocketArtifact]) -> dict[str
         packages = {}
         top_level_count = {}
         for artifact in sbom_artifacts:
-            package = Package.from_socket_artifact(asdict(artifact))
+            package = Package.from_diff_artifact(artifact.__dict__)
             if package.id in packages:
                 print("Duplicate package?")
             else:
@@ -534,22 +534,44 @@ def update_package_values(pkg: Package) -> Package:
         pkg.url += f"/{pkg.name}/overview/{pkg.version}"
         return pkg
 
-    def get_added_and_removed_packages(self, head_full_scan_id: str, new_full_scan_id: str) -> Tuple[Dict[str, Package], Dict[str, Package]]:
+    def get_added_and_removed_packages(
+            self,
+            head_full_scan_id: str,
+            new_full_scan_id: str,
+            merge: bool = False,
+            external_href: str = None,
+    ) -> Tuple[Dict[str, Package], Dict[str, Package], str, str]:
         """
         Get packages that were added and removed between scans.
 
         Args:
-            head_full_scan: Previous scan (may be None if first scan)
-            head_full_scan_id: New scan just created
-
+            head_full_scan_id: Previous scan
+            new_full_scan_id: New scan just created
+            merge: Whether the scan is merged into the default branch
+            external_href: External reference
         Returns:
             Tuple of (added_packages, removed_packages) dictionaries
         """
 
         log.info(f"Comparing scans - Head scan ID: {head_full_scan_id}, New scan ID: {new_full_scan_id}")
         diff_start = time.time()
         try:
-            diff_report = self.sdk.fullscans.stream_diff(self.config.org_slug, head_full_scan_id, new_full_scan_id, use_types=True).data
+            params = {
+                "before": head_full_scan_id,
+                "after": new_full_scan_id,
+                "description": f"Diff scan between head {head_full_scan_id} and new {new_full_scan_id} scans",
+                "merge": merge,
+            }
+            if external_href:
+                params["external_href"] = external_href
+            new_diff_scan = self.sdk.diffscans.create_from_ids(self.config.org_slug, params)
+            data = new_diff_scan.get("diff_scan", {})
+            diff_scan_id = data.get("id")
+            if not diff_scan_id:
+                log.error(f"Failed to get diff scan ID for {new_full_scan_id}")
+                log.error(new_diff_scan)
+                sys.exit(1)
+            diff_report = self.sdk.diffscans.get(self.config.org_slug, diff_scan_id)
         except APIFailure as e:
             log.error(f"API Error: {e}")
             sys.exit(1)
@@ -559,44 +581,63 @@ def get_added_and_removed_packages(self, head_full_scan_id: str, new_full_scan_i
             log.error(f"Stack trace:\n{traceback.format_exc()}")
             raise
 
+        diff_data = diff_report.get("diff_scan", {})
         diff_end = time.time()
+        diff_url = diff_data.get("html_url")
+        after_data = diff_data.get("after_full_scan")
+        if after_data:
+            new_full_scan_url = after_data.get("html_url")
+        else:
+            new_full_scan_url = ""
+        artifacts = diff_data.get("artifacts", {})
+        added = artifacts.get("added", [])
+        removed = artifacts.get("removed", [])
+        unchanged = artifacts.get("unchanged", [])
+        replaced = artifacts.get("replaced", [])
+        updated = artifacts.get("updated", [])
         log.info(f"Diff Report Gathered in {diff_end - diff_start:.2f} seconds")
         log.info("Diff report artifact counts:")
-        log.info(f"Added: {len(diff_report.artifacts.added)}")
-        log.info(f"Removed: {len(diff_report.artifacts.removed)}")
-        log.info(f"Unchanged: {len(diff_report.artifacts.unchanged)}")
-        log.info(f"Replaced: {len(diff_report.artifacts.replaced)}")
-        log.info(f"Updated: {len(diff_report.artifacts.updated)}")
+        log.info(f"Added: {len(added)}")
+        log.info(f"Removed: {len(removed)}")
+        log.info(f"Unchanged: {len(unchanged)}")
+        log.info(f"Replaced: {len(replaced)}")
+        log.info(f"Updated: {len(updated)}")
 
-        added_artifacts = diff_report.artifacts.added + diff_report.artifacts.updated
-        removed_artifacts = diff_report.artifacts.removed + diff_report.artifacts.replaced
+        added_artifacts = added + updated
+        removed_artifacts = removed
 
         added_packages: Dict[str, Package] = {}
         removed_packages: Dict[str, Package] = {}
 
         for artifact in added_artifacts:
+            artifact_id = artifact.get("id")
+            artifact_name = artifact.get("name")
+            artifact_version = artifact.get("version")
             try:
-                pkg = Package.from_diff_artifact(asdict(artifact))
+                pkg = Package.from_diff_artifact(artifact)
                 pkg = Core.update_package_values(pkg)
-                added_packages[artifact.id] = pkg
+                added_packages[pkg.id] = pkg
             except KeyError:
-                log.error(f"KeyError: Could not create package from added artifact {artifact.id}")
-                log.error(f"Artifact details - name: {artifact.name}, version: {artifact.version}")
+                log.error(f"KeyError: Could not create package from added artifact {artifact_id}")
+                log.error(f"Artifact details - name: {artifact_name}, version: {artifact_version}")
                 log.error("No matching packages found in new_full_scan")
 
         for artifact in removed_artifacts:
+            artifact_id = artifact.get("id")
+            artifact_name = artifact.get("name")
+            artifact_version = artifact.get("version")
             try:
-                pkg = Package.from_diff_artifact(asdict(artifact))
+                pkg = Package.from_diff_artifact(artifact)
                 pkg = Core.update_package_values(pkg)
                 if pkg.namespace:
                     pkg.purl += f"{pkg.namespace}/{pkg.purl}"
-                removed_packages[artifact.id] = pkg
+                removed_packages[pkg.id] = pkg
             except KeyError:
-                log.error(f"KeyError: Could not create package from removed artifact {artifact.id}")
-                log.error(f"Artifact details - name: {artifact.name}, version: {artifact.version}")
+                log.error(f"KeyError: Could not create package from removed artifact {artifact_id}")
+                log.error(f"Artifact details - name: {artifact_name}, version: {artifact_version}")
                 log.error("No matching packages found in head_full_scan")
 
-        return added_packages, removed_packages
+        return added_packages, removed_packages, diff_url, new_full_scan_url
 
     def create_new_diff(
             self,
@@ -642,7 +683,6 @@ def create_new_diff(
         try:
             new_scan_start = time.time()
             new_full_scan = self.create_full_scan(files_for_sending, params)
-            new_full_scan.sbom_artifacts = self.get_sbom_data(new_full_scan.id)
             new_scan_end = time.time()
             log.info(f"Total time to create new full scan: {new_scan_end - new_scan_start:.2f}")
         except APIFailure as e:
@@ -654,26 +694,15 @@ def create_new_diff(
             log.error(f"Stack trace:\n{traceback.format_exc()}")
             raise
 
-        scans_ready = self.check_full_scans_status(head_full_scan_id, new_full_scan.id)
-        if scans_ready is False:
-            log.error(f"Full scans did not complete within {self.config.timeout} seconds")
-        added_packages, removed_packages = self.get_added_and_removed_packages(head_full_scan_id, new_full_scan.id)
+        added_packages, removed_packages, diff_url, report_url = self.get_added_and_removed_packages(
+            head_full_scan_id,
+            new_full_scan.id
+        )
 
         diff = self.create_diff_report(added_packages, removed_packages)
-
-        base_socket = "https://socket.dev/dashboard/org"
         diff.id = new_full_scan.id
-
-        report_url = f"{base_socket}/{self.config.org_slug}/sbom/{diff.id}"
-        if not params.include_license_details:
-            report_url += "?include_license_details=false"
         diff.report_url = report_url
-
-        if head_full_scan_id is not None:
-            diff.diff_url = f"{base_socket}/{self.config.org_slug}/diff/{head_full_scan_id}/{diff.id}"
-        else:
-            diff.diff_url = diff.report_url
-
+        diff.diff_url = diff_url
         return diff
 
     def create_diff_report(

socketsecurity/core/classes.py

@@ -97,7 +97,7 @@ class AlertCounts(TypedDict):
     low: int
 
 @dataclass(kw_only=True)
-class Package(SocketArtifactLink):
+class Package():
     """
     Represents a package detected in a Socket Security scan.
     
@@ -106,16 +106,23 @@ class Package(SocketArtifactLink):
     """
 
     # Common properties from both artifact types
-    id: str
+    type: str
     name: str
     version: str
-    type: str
+    release: str
+    diffType: str
+    id: str
+    author: List[str] = field(default_factory=list)
     score: SocketScore
     alerts: List[SocketAlert]
-    author: List[str] = field(default_factory=list)
     size: Optional[int] = None
     license: Optional[str] = None
     namespace: Optional[str] = None
+    topLevelAncestors: Optional[List[str]] = None
+    direct: Optional[bool] = False
+    manifestFiles: Optional[List[SocketManifestReference]] = None
+    dependencies: Optional[List[str]] = None
+    artifact: Optional[SocketArtifactLink] = None
 
     # Package-specific fields
     license_text: str = ""
@@ -203,7 +210,9 @@ def from_diff_artifact(cls, data: dict) -> "Package":
             manifestFiles=ref.get("manifestFiles", []),
             dependencies=ref.get("dependencies"),
             artifact=ref.get("artifact"),
-            namespace=data.get('namespace', None)
+            namespace=data.get('namespace', None),
+            release=ref.get("release", None),
+            diffType=ref.get("diffType", None),
         )
 
 class Issue:

socketsecurity/core/helper/__init__.py

@@ -0,0 +1,119 @@
+import markdown
+from bs4 import BeautifulSoup, NavigableString, Tag
+import string
+
+
+class Helper:
+    @staticmethod
+    def parse_gfm_section(html_content):
+        """
+        Parse a GitHub-Flavored Markdown section containing a table and surrounding content.
+        Returns a dict with "before_html", "columns", "rows_html", and "after_html".
+        """
+        html = markdown.markdown(html_content, extensions=['extra'])
+        soup = BeautifulSoup(html, "html.parser")
+
+        table = soup.find('table')
+        if not table:
+            # If no table, treat entire content as before_html
+            return {"before_html": html, "columns": [], "rows_html": [], "after_html": ''}
+
+        # Collect HTML before the table
+        before_parts = [str(elem) for elem in table.find_previous_siblings()]
+        before_html = ''.join(reversed(before_parts))
+
+        # Collect HTML after the table
+        after_parts = [str(elem) for elem in table.find_next_siblings()]
+        after_html = ''.join(after_parts)
+
+        # Extract table headers
+        headers = [th.get_text(strip=True) for th in table.find_all('th')]
+
+        # Extract table rows (skip header)
+        rows_html = []
+        for tr in table.find_all('tr')[1:]:
+            cells = [str(td) for td in tr.find_all('td')]
+            rows_html.append(cells)
+
+        return {
+            "before_html": before_html,
+            "columns": headers,
+            "rows_html": rows_html,
+            "after_html": after_html
+        }
+
+    @staticmethod
+    def parse_cell(html_td):
+        """Convert a table cell HTML into plain text or a dict for links/images."""
+        soup = BeautifulSoup(html_td, "html.parser")
+        a = soup.find('a')
+        if a:
+            cell = {"url": a.get('href', '')}
+            img = a.find('img')
+            if img:
+                cell.update({
+                    "img_src": img.get('src', ''),
+                    "title": img.get('title', ''),
+                    "link_text": a.get_text(strip=True)
+                })
+            else:
+                cell["link_text"] = a.get_text(strip=True)
+            return cell
+        return soup.get_text(strip=True)
+
+    @staticmethod
+    def parse_html_parts(html_fragment):
+        """
+        Convert an HTML fragment into a list of parts.
+        Each part is either:
+          - {"text": "..."}
+          - {"link": "url", "text": "..."}
+          - {"img_src": "url", "alt": "...", "title": "..."}
+        """
+        soup = BeautifulSoup(html_fragment, 'html.parser')
+        parts = []
+
+        def handle_element(elem):
+            if isinstance(elem, NavigableString):
+                text = str(elem).strip()
+                if text and not all(ch in string.punctuation for ch in text):
+                    parts.append({"text": text})
+            elif isinstance(elem, Tag):
+                if elem.name == 'a':
+                    href = elem.get('href', '')
+                    txt = elem.get_text(strip=True)
+                    parts.append({"link": href, "text": txt})
+                elif elem.name == 'img':
+                    parts.append({
+                        "img_src": elem.get('src', ''),
+                        "alt": elem.get('alt', ''),
+                        "title": elem.get('title', '')
+                    })
+                else:
+                    # Recurse into children for nested tags
+                    for child in elem.children:
+                        handle_element(child)
+
+        for element in soup.contents:
+            handle_element(element)
+
+        return parts
+
+    @staticmethod
+    def section_to_json(section_result):
+        """
+        Convert a parsed section into structured JSON.
+        Returns {"before": [...], "table": [...], "after": [...]}.
+        """
+        # Build JSON rows for the table
+        table_rows = []
+        cols = section_result.get('columns', [])
+        for row_html in section_result.get('rows_html', []):
+            cells = [Helper.parse_cell(cell_html) for cell_html in row_html]
+            table_rows.append(dict(zip(cols, cells)))
+
+        return {
+            "before": Helper.parse_html_parts(section_result.get('before_html', '')),
+            "table": table_rows,
+            "after": Helper.parse_html_parts(section_result.get('after_html', ''))
+        }

socketsecurity/core/messages.py

@@ -292,7 +292,8 @@ def create_security_comment_json(diff: Diff) -> dict:
         output = {
             "scan_failed": scan_failed,
             "new_alerts": [],
-            "full_scan_id": diff.id
+            "full_scan_id": diff.id,
+            "diff_url": diff.diff_url
         }
         for alert in diff.new_alerts:
             alert: Issue

socketsecurity/output.py

@@ -66,7 +66,8 @@ def output_console_comments(self, diff_report: Diff, sbom_file_name: Optional[st
 
         console_security_comment = Messages.create_console_security_alert_table(diff_report)
         self.logger.info("Security issues detected by Socket Security:")
-        self.logger.info(console_security_comment)
+        self.logger.info(f"Diff Url: {diff_report.diff_url}")
+        self.logger.info(f"\n{console_security_comment}")
 
     def output_console_json(self, diff_report: Diff, sbom_file_name: Optional[str] = None) -> None:
         """Outputs JSON formatted results"""