[uss_qualifier] Generate dynamic test documentation before scenario execution

.basedpyright/baseline.json

@@ -21135,54 +21135,6 @@
             }
         ],
         "./monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py": [
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 54,
-                    "endColumn": 59,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            },
             {
                 "code": "reportOptionalMemberAccess",
                 "range": {
@@ -21504,32 +21456,6 @@
                 }
             }
         ],
-        "./monitoring/uss_qualifier/scenarios/interuss/geospatial_map/geospatial_feature_comprehension.py": [
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 40,
-                    "endColumn": 45,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 46,
-                    "endColumn": 51,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 39,
-                    "endColumn": 44,
-                    "lineCount": 1
-                }
-            }
-        ],
         "./monitoring/uss_qualifier/scenarios/interuss/mock_uss/configure_locality.py": [
             {
                 "code": "reportOptionalMemberAccess",
@@ -21747,22 +21673,6 @@
                     "lineCount": 1
                 }
             },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 65,
-                    "endColumn": 70,
-                    "lineCount": 1
-                }
-            },
-            {
-                "code": "reportOptionalMemberAccess",
-                "range": {
-                    "startColumn": 140,
-                    "endColumn": 144,
-                    "lineCount": 1
-                }
-            },
             {
                 "code": "reportOptionalMemberAccess",
                 "range": {

monitoring/uss_qualifier/configurations/dev/general_flight_auth.jsonnet

@@ -36,6 +36,7 @@ local validation = import './library/validation.yaml';
     artifacts: {
       raw_report: {},
       sequence_view: {},
+      globally_expanded_report: {},
       tested_requirements: [
         {
           report_name: 'requirements',

monitoring/uss_qualifier/configurations/dev/geospatial_comprehension.yaml

@@ -37,5 +37,6 @@ v1:
   artifacts:
     raw_report: {}
     sequence_view: {}
+    globally_expanded_report: {}
   validation:
     $ref: ./library/validation.yaml#/normal_test

monitoring/uss_qualifier/configurations/dev/library/resources.yaml

@@ -485,12 +485,12 @@ example_feature_check_table:
                 use_timezone: +01:00
               duration: 5m
           expected_result: Advise
-        - geospatial_check_id: TEST_002
+        - geospatial_check_id: TEST_003
           requirement_ids:
             - REQ_001
             - REQ_003
             - REQ_004
-          description: The second test step defined by the test designer
+          description: The third test step defined by the test designer
           operation_rule_set: Rules1
           restriction_source: ThisRegulator
           volumes:

monitoring/uss_qualifier/scenarios/interuss/flight_authorization/general_flight_authorization.py

@@ -12,6 +12,7 @@
 )
 from monitoring.monitorlib.temporal import Time, TimeDuringTest
 from monitoring.uss_qualifier.configurations.configuration import ParticipantID
+from monitoring.uss_qualifier.requirements.definitions import RequirementID
 from monitoring.uss_qualifier.resources.flight_planning import (
     FlightIntentsResource,
     FlightPlannerResource,
@@ -25,7 +26,9 @@
     FlightCheckTableResource,
 )
 from monitoring.uss_qualifier.scenarios.documentation.definitions import (
+    TestCaseDocumentation,
     TestCheckDocumentation,
+    TestScenarioDocumentation,
     TestStepDocumentation,
 )
 from monitoring.uss_qualifier.scenarios.scenario import TestScenario
@@ -63,35 +66,24 @@ def __init__(
         self.participant_id = planner.participant_id
         self.flight_intents = flight_intents.get_flight_intents()
 
-    def run(self, context: ExecutionContext):
-        self.begin_test_scenario(context)
-        times = {
-            TimeDuringTest.StartOfTestRun: Time(context.start_time),
-            TimeDuringTest.StartOfScenario: Time(arrow.utcnow().datetime),
-        }
+    def _rewrite_documentation(self):
+        """The documentation in the standard, static accompanying .md file acts as a template, but the test scenario
+        dynamically adjusts its test procedure based on the FlightCheckTable provided in configuration."""
+        original_case = self.documentation.cases[0]
+        original_step = original_case.steps[0]
+        steps = []
 
-        self.begin_test_case("Flight planning")
-        self._plan_flights(times)
-        self.end_test_case()
-
-        self.end_test_scenario()
-
-    def _plan_flights(self, times: dict[TimeDuringTest, Time]):
         for row in self.table.rows:
             # Collect checks applicable to this row/test step
             checks = [
-                _get_check_by_name(self._current_case.steps[0], name)
+                _get_check_by_name(original_step, name)
                 for name in (_VALID_API_RESPONSE_NAME, _SUCCESSFUL_CLOSURE_NAME)
             ]
             if row.acceptance_expectation == AcceptanceExpectation.MustBeAccepted:
-                acceptance_check = _get_check_by_name(
-                    self._current_case.steps[0], _ACCEPT_CHECK_NAME
-                )
+                acceptance_check = _get_check_by_name(original_step, _ACCEPT_CHECK_NAME)
                 checks.append(acceptance_check)
             elif row.acceptance_expectation == AcceptanceExpectation.MustBeRejected:
-                rejection_check = _get_check_by_name(
-                    self._current_case.steps[0], _REJECT_CHECK_NAME
-                )
+                rejection_check = _get_check_by_name(original_step, _REJECT_CHECK_NAME)
                 checks.append(rejection_check)
             elif row.acceptance_expectation == AcceptanceExpectation.Irrelevant:
                 pass  # No acceptance-related checks to perform in this case
@@ -102,12 +94,12 @@ def _plan_flights(self, times: dict[TimeDuringTest, Time]):
 
             if row.conditions_expectation == ConditionsExpectation.MustBePresent:
                 conditional_check = _get_check_by_name(
-                    self._current_case.steps[0], _CONDITIONAL_CHECK_NAME
+                    original_step, _CONDITIONAL_CHECK_NAME
                 )
                 checks.append(conditional_check)
             elif row.conditions_expectation == ConditionsExpectation.MustBeAbsent:
                 unconditional_check = _get_check_by_name(
-                    self._current_case.steps[0], _UNCONDITIONAL_CHECK_NAME
+                    original_step, _UNCONDITIONAL_CHECK_NAME
                 )
                 checks.append(unconditional_check)
             elif row.conditions_expectation == ConditionsExpectation.Irrelevant:
@@ -118,26 +110,61 @@ def _plan_flights(self, times: dict[TimeDuringTest, Time]):
                 )
 
             # Construct documentation for this test step
-            # Note that we are duck-typing a List[str] into a List[RequirementID] for applicable_requirements, but this
-            # should be ok as the requirements are only used as strings from this point.
             step_checks = [
                 TestCheckDocumentation(
                     name=c.name,
                     url=c.url,
-                    applicable_requirements=row.requirement_ids,
+                    applicable_requirements=[
+                        RequirementID(r) for r in row.requirement_ids
+                    ],
                     has_todo=c.has_todo,
                     severity=c.severity,
                 )
                 for c in checks
             ]
-            doc = TestStepDocumentation(
-                name=row.flight_check_id,
-                url=self._current_case.steps[0].url,
-                checks=step_checks,
+            steps.append(
+                TestStepDocumentation(
+                    name=row.flight_check_id,
+                    url=original_step.url,
+                    checks=step_checks,
+                )
             )
 
-            # Officially begin the test step
-            self.begin_dynamic_test_step(doc)
+        case = TestCaseDocumentation(
+            name=original_case.name,
+            url=original_case.url,
+            steps=steps,
+        )
+
+        new_doc = TestScenarioDocumentation(
+            name=self.documentation.name,
+            url=self.documentation.url,
+            local_path=self.documentation.local_path,
+            cases=[case],
+        )
+        if "resources" in self.documentation:
+            new_doc.resources = self.documentation.resources
+        if "cleanup" in self.documentation:
+            new_doc.cleanup = self.documentation.cleanup
+        self.documentation = new_doc
+
+    def run(self, context: ExecutionContext):
+        self._rewrite_documentation()
+        self.begin_test_scenario(context)
+        times = {
+            TimeDuringTest.StartOfTestRun: Time(context.start_time),
+            TimeDuringTest.StartOfScenario: Time(arrow.utcnow().datetime),
+        }
+
+        self.begin_test_case("Flight planning")
+        self._plan_flights(times)
+        self.end_test_case()
+
+        self.end_test_scenario()
+
+    def _plan_flights(self, times: dict[TimeDuringTest, Time]):
+        for row in self.table.rows:
+            self.begin_test_step(row.flight_check_id)
 
             # Attempt planning action
             times[TimeDuringTest.TimeOfEvaluation] = Time(arrow.utcnow().datetime)