Implement Categorization for WebVTT

Implement rudimentary categorization system and add a README.

Author: BenjaminSchaaf

webvtt/README.md

@@ -0,0 +1,14 @@
+# WebVTT Tests
+
+## Categorization
+
+Dependencies:
+* Python 3
+
+Once you have run the tests with `/tools/runner/index.html`, press the "Download
+JSON results" button to download a file containing the results. You can then run
+the categorization tool to get a better overview of the test results:
+
+```bash
+$ python3 tools/categorize_results.py runner-results.json
+```

webvtt/api/VTTCue/categories.json

@@ -0,0 +1,3 @@
+{
+    "region.html": "regions"
+}

webvtt/api/categories.json

@@ -0,0 +1,5 @@
+{
+    ":subcategories": ["VTTCue/categories.json"],
+    "VTTCue/*": "cues",
+    "VTTRegion/*": "regions"
+}

webvtt/categories.json

@@ -0,0 +1,12 @@
+{
+    ":categories": [
+        ["api", "parsing", "rendering"],
+        ["cues"],
+        ["regions"],
+        ["stylesheets"]
+    ],
+    ":subcategories": ["api/categories.json", "parsing/categories.json"],
+    "api/*": "api",
+    "parsing/*": "parsing",
+    "rendering/*": "rendering"
+}

webvtt/parsing/categories.json

@@ -0,0 +1,8 @@
+{
+    ":categories": [
+        ["file-parsing", "cue-text-parsing"]
+    ],
+    ":subcategories": ["file-parsing/categories.json"],
+    "file-parsing/*": "file-parsing",
+    "cue-text-parsing/*": "cue-text-parsing"
+}

webvtt/parsing/file-parsing/categories.json

@@ -0,0 +1,6 @@
+{
+    ":categories": [
+        ["file-parsing", "cue-text-parsing"]
+    ],
+    ":subcategories": ["tests/categories.json"]
+}

webvtt/parsing/file-parsing/tests/categories.json

@@ -0,0 +1,9 @@
+{
+    "arrows.html": "cues",
+    "ids.html": "cues",
+    "settings-*.html": "cues",
+    "signature-timings.html": "cues",
+    "timings-*.html": "cues",
+    "regions-*.html": "regions",
+    "stylesheets-*.html": "stylesheets"
+}

webvtt/tools/categorize_results.py

@@ -0,0 +1,137 @@
+import os
+import sys
+import json
+import fnmatch
+
+TEST_DIR = "/webvtt/"
+CATEGORIES_FILE = "../categories.json"
+
+class Test:
+    def __init__(self, file, name, status, message):
+        self.file = file
+        self.name = name
+        self.status = status
+        self.message = message
+        self.passed = status == 'PASS'
+        self.categories = []
+
+    @classmethod
+    def from_json(cls, json):
+        file = json["test"]
+        if not file.startswith(TEST_DIR):
+            return []
+        file = file[len(TEST_DIR):]
+
+        status = json["status"]
+        message = json["message"]
+
+        tests = []
+
+        for test in json["subtests"]:
+            name = test["name"]
+            if status == 'OK':
+                test_status = test["status"]
+                test_message = test["message"]
+            else:
+                test_status, test_message = status, message
+
+            tests.append(Test(file, name, test_status, test_message))
+
+        return tests
+
+class Category:
+    def __init__(self, names):
+        self.names = set(names)
+        self.tests = {}
+
+    @classmethod
+    def from_json(cls, json):
+        return Category(json)
+
+    def add_test(self, name, test):
+        self.tests[test] = name
+
+    def __contains__(self, name):
+        return name in self.names
+
+def parse_results(file):
+    data = json.load(file)
+
+    results = data["results"]
+    tests = []
+    for result in results:
+        tests += Test.from_json(result)
+
+    return tests
+
+def parse_categories(file, tests, categories = None, categories_map = None):
+    data = json.load(file)
+    basepath = os.path.dirname(file.name)
+
+    categories = categories or []
+
+    if categories_map:
+        categories_map = dict(categories_map)
+    else:
+        categories_map = {}
+
+    if ":categories" in data:
+        for cat_data in data[":categories"]:
+            category = Category.from_json(cat_data)
+
+            categories.append(category)
+            for name in category.names:
+                categories_map[name] = category
+
+    for pattern, category_name in data.items():
+        if pattern.startswith(":"):
+            continue
+        category = categories_map[category_name]
+
+        file_pattern = os.path.normpath(os.path.join(basepath, pattern))
+        for test in tests:
+            if fnmatch.fnmatch(test.name, file_pattern) or fnmatch.fnmatch(test.file, file_pattern):
+                category.add_test(category_name, test)
+                test.categories.append(category)
+
+    if ":subcategories" in data:
+        for subcat_name in data[":subcategories"]:
+            path = os.path.join(basepath, subcat_name)
+            file = open(path, "r")
+            parse_categories(file, tests, categories, categories_map)
+
+    return categories
+
+def main(argv):
+    if len(argv) == 1:
+        if argv[0] == '-':
+            results_file = sys.stdin
+        else:
+            results_file = open(argv[0], "r")
+    else:
+        print("USAGE: python3 categorize_results.py <file>")
+        print("<file>\tA file containing wpt results. Or `-` for reading results from stdin.")
+        return
+
+    filepath = os.path.dirname(__file__)
+    categories_path = os.path.join(filepath, CATEGORIES_FILE)
+    categories_file = open(categories_path, "r")
+
+    tests = parse_results(results_file)
+    categories = parse_categories(categories_file, tests)
+
+    for category in categories:
+        tests_by_name = { name: [] for name in category.names }
+        for test, name in category.tests.items():
+            tests_by_name[name].append(test)
+
+        for name in category.names:
+            test_group = tests_by_name[name]
+            amount = len(test_group)
+            if amount == 0:
+                continue
+            passed = sum(1 for test in test_group if test.passed)
+            print("{}:\t{}/{} - {}%".format(name, passed, amount, round(passed / amount * 100, 2)))
+
+if __name__ == "__main__":
+    main(sys.argv[1:])