|
| 1 | +#!/usr/bin/env python3 |
| 2 | +""" |
| 3 | + Module to verify VTR test suites run by the GitHub CI. |
| 4 | +
|
| 5 | + Test suites in VTR are verified by ensuring that all tasks in the test suite |
| 6 | + appear in the task list and that there are no tasks in the task list which |
| 7 | + are not in the test suite. |
| 8 | +
|
| 9 | + A JSON file is used to tell this module which test suites to verify. |
| 10 | +
|
| 11 | + This module is designed to be used within the CI of VTR to ensure that tasks |
| 12 | + within test suites are running all the tasks they are intended to. |
| 13 | +""" |
| 14 | +import os |
| 15 | +import argparse |
| 16 | +import json |
| 17 | +import sys |
| 18 | +from dataclasses import dataclass, field |
| 19 | +from typing import List, Set |
| 20 | +from pathlib import Path |
| 21 | + |
| 22 | + |
| 23 | +@dataclass(order=True, frozen=True) |
| 24 | +class TestSuite: |
| 25 | + """ |
| 26 | + Data class used to store information about a test suite. |
| 27 | + """ |
| 28 | + |
| 29 | + name: str |
| 30 | + ignored_tasks: List[str] = field(default_factory=list) |
| 31 | + |
| 32 | + |
| 33 | +def parse_test_suite_info(test_suite_info_file: str) -> List[TestSuite]: |
| 34 | + """ |
| 35 | + Parses the given test_suite_info file. The test suite info file is expected |
| 36 | + to be a JSON file which contains information on which test suites in the |
| 37 | + regression tests to verify and if any of the tasks should be ignored. |
| 38 | +
|
| 39 | + The JSON should have the following form: |
| 40 | + {"test_suites": [ |
| 41 | + { |
| 42 | + "name": "<test_suite_name>", |
| 43 | + "ignored_tasks": [ |
| 44 | + "<ignored_task_name>", |
| 45 | + ... |
| 46 | + ] |
| 47 | + }, |
| 48 | + { |
| 49 | + ... |
| 50 | + } |
| 51 | + ]} |
| 52 | + """ |
| 53 | + with open(test_suite_info_file, "r") as file: |
| 54 | + data = json.load(file) |
| 55 | + |
| 56 | + assert isinstance(data, dict), "Test suite info should be a dictionary" |
| 57 | + assert "test_suites" in data, "A list of test suites must be provided" |
| 58 | + |
| 59 | + test_suites = [] |
| 60 | + for test_suite in data["test_suites"]: |
| 61 | + assert isinstance(test_suite, dict), "Test suite should be a dictionary" |
| 62 | + assert "name" in test_suite, "All test suites must have names" |
| 63 | + assert "ignored_tasks" in test_suite, "All test suite must have an ignored task list" |
| 64 | + |
| 65 | + test_suites.append( |
| 66 | + TestSuite( |
| 67 | + name=test_suite["name"], |
| 68 | + ignored_tasks=test_suite["ignored_tasks"], |
| 69 | + ) |
| 70 | + ) |
| 71 | + |
| 72 | + return test_suites |
| 73 | + |
| 74 | + |
| 75 | +def parse_task_list(task_list_file: str) -> Set[str]: |
| 76 | + """ |
| 77 | + Parses the given task_list file and returns a list of the tasks within |
| 78 | + the task list. |
| 79 | + """ |
| 80 | + tasks = set() |
| 81 | + with open(task_list_file, "r") as file: |
| 82 | + for line in file: |
| 83 | + # Strip the whitespace from the line. |
| 84 | + line.strip() |
| 85 | + # If this is a comment line, skip it. |
| 86 | + if line[0] == "#": |
| 87 | + continue |
| 88 | + # Split the line. This is used in case there is a comment on the line. |
| 89 | + split_line = line.split() |
| 90 | + if split_line: |
| 91 | + # If the line can be split (i.e. the line is not empty), add |
| 92 | + # the first part of the line to the tasks list, stripping any |
| 93 | + # trailing "/" characters. |
| 94 | + tasks.add(split_line[0].rstrip("/")) |
| 95 | + |
| 96 | + return tasks |
| 97 | + |
| 98 | + |
| 99 | +def get_expected_task_list(test_suite_dir: str, reg_tests_parent_dir: str) -> Set[str]: |
| 100 | + """ |
| 101 | + Get the expected task list by parsing the test suite directory and finding |
| 102 | + all files that look like config files. |
| 103 | + """ |
| 104 | + # Get all config files in the test suite. These will indicated where all |
| 105 | + # the tasks are in the suite. |
| 106 | + base_path = Path(test_suite_dir) |
| 107 | + assert base_path.is_dir() |
| 108 | + config_files = list(base_path.rglob("config.txt")) |
| 109 | + |
| 110 | + # Get a list of all the expected tasks in the task list |
| 111 | + expected_task_list = set() |
| 112 | + for config_file in config_files: |
| 113 | + config_dir = os.path.dirname(config_file) |
| 114 | + task_dir = os.path.dirname(config_dir) |
| 115 | + # All tasks in the task list are relative to the parent of the regression |
| 116 | + # tests directory. |
| 117 | + expected_task_list.add(os.path.relpath(task_dir, reg_tests_parent_dir)) |
| 118 | + |
| 119 | + return expected_task_list |
| 120 | + |
| 121 | + |
| 122 | +def verify_test_suite(test_suite: TestSuite, regression_tests_dir: str): |
| 123 | + """ |
| 124 | + Verifies the given test suite by looking into the regression tests directory |
| 125 | + for the suite and ensures that all expected tasks are present in the suite's |
| 126 | + task list. |
| 127 | +
|
| 128 | + Returns the number of failures found in the test suite. |
| 129 | + """ |
| 130 | + # Check that the test suite exists in the regression tests directory |
| 131 | + test_suite_dir = os.path.join(regression_tests_dir, test_suite.name) |
| 132 | + if not os.path.exists(test_suite_dir): |
| 133 | + print("\tError: Test suite not found in regression tests directory") |
| 134 | + return 1 |
| 135 | + |
| 136 | + # Get the expected tasks list from the test suite directory. |
| 137 | + reg_tests_parent_dir = os.path.dirname(regression_tests_dir.rstrip("/")) |
| 138 | + expected_task_list = get_expected_task_list(test_suite_dir, reg_tests_parent_dir) |
| 139 | + |
| 140 | + # Get the task list file from the test suite and parse it to get the actual |
| 141 | + # task list. |
| 142 | + task_list_file = os.path.join(test_suite_dir, "task_list.txt") |
| 143 | + if not os.path.exists(task_list_file): |
| 144 | + print("\tError: Test suite does not have a root-level task list") |
| 145 | + return 1 |
| 146 | + actual_task_list = parse_task_list(task_list_file) |
| 147 | + |
| 148 | + # Keep track of the number of failures |
| 149 | + num_failures = 0 |
| 150 | + |
| 151 | + # Process the ignored tests |
| 152 | + ignored_tasks = set() |
| 153 | + for ignored_task in test_suite.ignored_tasks: |
| 154 | + # Ignored tasks are relative to the test directory, get their full path. |
| 155 | + ignored_task_path = os.path.join(test_suite_dir, ignored_task) |
| 156 | + # Check that the task exists. |
| 157 | + if not os.path.exists(ignored_task_path): |
| 158 | + print(f"\tError: Ignored task '{ignored_task}' not found in test suite") |
| 159 | + num_failures += 1 |
| 160 | + continue |
| 161 | + # If the task exists, add it to the ignored tasks list relative to the |
| 162 | + # reg test's parent directory so it can be compared properly. |
| 163 | + ignored_tasks.add(os.path.relpath(ignored_task_path, reg_tests_parent_dir)) |
| 164 | + |
| 165 | + if len(ignored_tasks) > 0: |
| 166 | + print(f"\tWarning: {len(ignored_tasks)} tasks were ignored") |
| 167 | + |
| 168 | + # Check for any missing tasks in the task list |
| 169 | + for task in expected_task_list: |
| 170 | + # If this task is ignored, it is expected to be missing. |
| 171 | + if task in ignored_tasks: |
| 172 | + continue |
| 173 | + # If the task is not in the actual task list, this is an error. |
| 174 | + if task not in actual_task_list: |
| 175 | + print(f"\tError: Failed to find task '{task}' in task list!") |
| 176 | + num_failures += 1 |
| 177 | + |
| 178 | + # Check for any tasks in the task list which should not be there |
| 179 | + for task in actual_task_list: |
| 180 | + # If a task is in the task list, but is not in the test directory, this |
| 181 | + # is a failure. |
| 182 | + if task not in expected_task_list: |
| 183 | + print(f"\tError: Task '{task}' found in task list but not in test directory") |
| 184 | + num_failures += 1 |
| 185 | + # If a task is in the task list, but is marked as ignored, this must be |
| 186 | + # a mistake. |
| 187 | + if task in ignored_tasks: |
| 188 | + print(f"\tError: Task '{task}' found in task list but was marked as ignored") |
| 189 | + |
| 190 | + return num_failures |
| 191 | + |
| 192 | + |
| 193 | +def verify_test_suites(): |
| 194 | + """ |
| 195 | + Verify the VTR test suites. |
| 196 | +
|
| 197 | + Test suites are verified by checking the tasks within their test directory |
| 198 | + and the tasks within the task list at the root of that directory and ensuring |
| 199 | + that they match. If there are any tasks which appear in one but not the other, |
| 200 | + an error is produced and this script will return an error code. |
| 201 | + """ |
| 202 | + # Set up the argument parser object. |
| 203 | + parser = argparse.ArgumentParser(description="Verifies the test suites used in VTR.") |
| 204 | + parser.add_argument( |
| 205 | + "-vtr_regression_tests_dir", |
| 206 | + type=str, |
| 207 | + required=True, |
| 208 | + help="The path to the vtr_flow/tasks/regression_tests directory in VTR.", |
| 209 | + ) |
| 210 | + parser.add_argument( |
| 211 | + "-test_suite_info", |
| 212 | + type=str, |
| 213 | + required=True, |
| 214 | + help="Information on the test suite (must be a JSON file).", |
| 215 | + ) |
| 216 | + |
| 217 | + # Parse the arguments from the command line. |
| 218 | + args = parser.parse_args() |
| 219 | + |
| 220 | + # Verify each of the test suites. |
| 221 | + num_failures = 0 |
| 222 | + test_suites = parse_test_suite_info(args.test_suite_info) |
| 223 | + for test_suite in test_suites: |
| 224 | + print(f"Verifying test suite: {test_suite.name}") |
| 225 | + test_suite_failures = verify_test_suite(test_suite, args.vtr_regression_tests_dir) |
| 226 | + print(f"\tTest suite had {test_suite_failures} failures\n") |
| 227 | + num_failures += test_suite_failures |
| 228 | + |
| 229 | + # If any failures were found in any suite, return exit code 1. |
| 230 | + if num_failures != 0: |
| 231 | + print(f"Failure: Test suite verifcation failed with {num_failures} failures") |
| 232 | + print(f"Please fix the failing test suites found in {args.vtr_regression_tests_dir}") |
| 233 | + print(f"If necessary, update the test suites info found here: {args.test_suite_info}") |
| 234 | + sys.exit(1) |
| 235 | + |
| 236 | + print(f"Success: All test suites in {args.test_suite_info} passed") |
| 237 | + |
| 238 | + |
| 239 | +if __name__ == "__main__": |
| 240 | + verify_test_suites() |
0 commit comments