WIP: pre move stdname

In general works through getting minimal set of JSON to process.
About to move standard being tested into the parametrization id

github/branches/test_branch_protection.py

@@ -29,7 +29,7 @@ def test_required_protections(
     owner, repo = url.split("/")[3:5]
     rules = get_repo_branch_protections(gql_connection, f"{owner}/{repo}")
     if not rules:
-        assert False, f"ERROR: no branch protection for '{owner}/{repo}:{branch}'"
+        assert False, f"ERROR:SOGH001:{owner}/{repo}:{branch} has no branch protection"
     else:
         met, message = validate_compliance.validate_branch_protections(
             rules, branch, criteria

github/branches/validate_compliance.py

@@ -45,9 +45,7 @@ def meets_criteria(protections: List[BranchProtectionRule], criteria: str) -> bo
     elif criteria == "rule conflicts":
         met = all(r.rule_conflict_count == 0 for r in protections)
     else:
-        # raising assert works best when called from pytest, as we won't
-        # _always_ be called from pytest.
-        assert False, f"ERROR: no support for '{criteria}'"
+        met = False
     return met
 
 
@@ -72,12 +70,15 @@ def validate_branch_protections(
         # results.append(
         assert (
             False
-        ), f"ERROR: no branch protection for '{data.name_with_owner}:{branch}'"
+        ), f"ERROR:SOGH001:{data.name_with_owner}:{branch} has no branch protection"
         # )
     else:
         # see if at least one rule matches specified criteria
-        message = f"ERROR: no {criteria} for branch '{data.name_with_owner}:{branch}'"
+        message = (
+            f"ERROR:SOGH001:{data.name_with_owner}:{branch} has no {criteria} rule"
+        )
         return meets_criteria(active_rules, criteria), message
+        # vscode correctly tells me that all code below here is unreachable
         # see if at least one rule matches each criteria
         for criteria in required_criteria:
             if not meets_criteria(active_rules, criteria):
@@ -90,6 +91,7 @@ def validate_branch_protections(
                     f"FYI: no {criteria} for branch '{data.name_with_owner}:{branch}' (optional)"
                 )
 
+    # vscode correctly tells me that all code below here is unreachable
     # regardless of match, we'll also warn on conflicting rules
     for criteria in warning_criteria:
         if not meets_criteria(active_rules, criteria):

github/exemptions-github.yaml

@@ -0,0 +1,10 @@
+exemptions:
+  - test_name: "test_require_2fa"
+    test_param_id: "*-firefox-devtool"
+    reason: "transition"
+    expiration_day: 2050-01-01
+
+  - test_name: "test_require_2fa"
+    test_param_id: "*-mozilla-mobile"
+    reason: "test of xpass transition"
+    expiration_day: 2050-01-01

github/github.jq

@@ -0,0 +1,8 @@
+[.report.tests[] | select(.call.outcome != "passed")
+    | { full_name: .name,
+        modified_status: .outcome,
+        original_status: .metadata[0].outcome,
+        reason:(.call.xfail_reason // ""),
+        longrepr: .call.longrepr
+    }
+]

github/manage_issues.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# PYTHON_ARGCOMPLETE_OK
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+"""Perform actions based on frost results"""
+
+import argparse
+import json
+import re
+from dataclasses import dataclass
+from pprint import pprint
+from typing import List
+
+import argcomplete
+
+_epilog = ""
+
+# constants
+# Sample line
+SPEC_DECODER_RE = re.compile(
+    r"""
+    (?P<path>[^:]+):: # path
+    (?P<method>\w+)\[ # method
+    (?P<test_name>[^-]+)- # assumes no hyphen in test_name
+    (?P<param_id>[^]]+)\]
+""",
+    re.VERBOSE,
+)
+
+# Sample line:
+#  E   AssertionError: ERROR:SOGH003:firefox-devtools doesn't meet two factor required - required\n    assert False
+ASSERT_DECODER_RE = re.compile(
+    r"""
+    (E\s+AssertionError:)?\s*  # preamble
+    (?P<severity>[^:]+):
+    (?P<standard>[^:]+):
+    (?P<info>\S+)
+""",
+    re.VERBOSE | re.MULTILINE,
+)
+
+
+@dataclass
+class Action:
+    owner: str
+    repo: str
+    branch: str
+
+
+def parse_action_string(name: str) -> List[str]:
+    """
+    comment
+    """
+    matches = SPEC_DECODER_RE.match(name)
+    return matches.groups()
+
+
+def infer_resource_type(path: str) -> str:
+    """infer object type
+
+    This relies on the file system structure of the tests
+    We currently assume it is:
+        "github/" resource_type "/.*"
+    """
+    prefix = "github/"
+    start = path.find(prefix) + len(prefix)
+    end = path.find("/", start)
+    resource_type = path[start:end]
+    return resource_type
+
+
+def extract_standard(assert_msg: str) -> str:
+    """
+    pull standard(s) out of the assert string
+
+    TODO:
+    - support more than one result in string
+    """
+    for item in ASSERT_DECODER_RE.finditer(assert_msg):
+        pprint(item.groupdict()["standard"])
+
+
+def create_branch_action(action_spec: dict) -> Action:
+    """Parse pytest info into information needed to open an issue against a
+    specific branch"""
+
+    path, method, test_name, param_id = parse_action_string(action_spec["full_name"])
+    standard = extract_standard(action_spec["longrepr"])
+    url, branch = param_id.split(",")
+    owner, repo = url.split("/")[3:5]
+
+
+def create_org_action(action_spec: dict) -> Action:
+    """
+    Break out the org info from the json
+    """
+    found = False
+    for item in ASSERT_DECODER_RE.finditer(action_spec["longrepr"]):
+        pprint(item.groupdict())
+        found = True
+    if not found:
+        raise KeyError(f"Malformed json {repr(action_spec)}")
+
+
+def create_action_spec(action_spec: dict) -> Action:
+    # for now, just return Action -- later decode may involve inferring what to
+    # do ("xpass" detection)
+    name = action_spec["full_name"]
+    path, _, _, _ = parse_action_string(name)
+    resource_type = infer_resource_type(path)
+    if resource_type == "orgs":
+        action = create_org_action(action_spec)
+    elif resource_type == "branches":
+        action = create_branch_action(action_spec)
+    else:
+        raise TypeError(f"unknown resource type '{resource_type}' from '{name}")
+
+    # full name is file_path::method[test_name-parametrize_id]
+    pprint(action_spec)
+    return action
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__, epilog=_epilog)
+    # parser.add_argument(
+    #     "--debug", action="store_true", help="include dump of all data returned"
+    # )
+    # parser.add_argument("--owners", action="store_true", help="Also show owners")
+    # parser.add_argument("--email", action="store_true", help="include owner email")
+    # parser.add_argument(
+    #     "--all-my-orgs",
+    #     action="store_true",
+    #     help="act on all orgs for which you're an owner",
+    # )
+    # parser.add_argument(
+    #     "--names-only",
+    #     action="store_true",
+    #     help="Only output your org names for which you're an owner",
+    # )
+    parser.add_argument("json_file", help="frost json output")
+    argcomplete.autocomplete(parser)
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    with open(args.json_file, "r") as jf:
+        issue_actions = json.loads(jf.read())
+
+    print(f"Processing {len(issue_actions)}")
+    for action_spec in issue_actions:
+        action = create_action_spec(action_spec)