tool_tester: make a very simple pngcmp tester

Author: angelosa

regtests/assets/png/black320x240.png

@@ -6,9 +6,8 @@
 import sys
 from os.path import join, dirname, realpath
 import logging
-from tool_tester.unidasm import (
-    UnidasmTests
-)
+from tool_tester.pngcmp import PngCmpTests
+from tool_tester.unidasm import UnidasmTests
 
 if __name__ == "__main__":
     # TODO: add colorized messages
@@ -23,7 +22,7 @@
     chained_results = []
     # TODO: point to $(regtests)\assets, configure if necessary
     assets_folder = join(dirname(dirname(realpath(__file__))), "assets")
-    for test_cls in [UnidasmTests]:
+    for test_cls in [PngCmpTests, UnidasmTests]:
         test_fn = test_cls(assets_folder)
         logging.info("Start test suite: %s", test_fn.identifier)
         chained_results.append(test_fn.execute_tests(test_fn.compose_tests()))

regtests/assets/png/white320x240.png

@@ -56,7 +56,9 @@ def execute_tests(self, test_pool: Dict) -> bool:
                     __launch_fullpath, 
                     encoding="utf-8",
                     capture_output=True,
-                    check=True,
+                    # TODO: we need to disable check here cause pngcmp has a returncode of 1 when diverging snapshots occurs (wtf)
+                    #check=True,
+                    check=False,
                     text=True,
                     shell=False
                 )

regtests/tool_tester/test_tools.py

@@ -0,0 +1,48 @@
+##
+## license:BSD-3-Clause
+## copyright-holders:Angelo Salese
+##
+
+import os
+from subprocess import CompletedProcess
+#from dataclasses import dataclass
+#import logging
+from tool_tester._selfexe import SelfExeTests
+from typing import List
+
+class PngCmpTests(SelfExeTests):
+    """A very bread and butter test suite against PngCmp tool
+
+    * same_* tests if file A is equal, by checking if A == A
+    * divergent_* tests if A != B, and B != A
+
+    We don't need to actually save the output to anywhere else,
+    this tool will most likely be reused in other areas anyway. ;)
+
+    Args:
+        SelfExeTests ([type]): [description]
+    """
+
+    def __init__(self, assets_path: str):
+        super().__init__("pngcmp", assets_path)
+        self._png_test_folder = os.path.join(assets_path, "png")
+
+    def _collect_tests(self):
+        return {
+            "same_black": ["black320x240", "black320x240"],
+            "same_white": ["white320x240", "white320x240"],
+            "divergent_black_on_white": ["black320x240", "white320x240"],
+            "divergent_white_on_black": ["white320x240", "black320x240"]
+        }
+
+    def _subprocess_args(self, test_handle: str, test_params: List):
+        return [
+            os.path.join(self._png_test_folder, test_params[0] + ".png"), 
+            os.path.join(self._png_test_folder, test_params[1] + ".png"),
+            os.devnull
+        ]
+
+    def _assert_test_result(self, launch_result: CompletedProcess, test_handle: str, test_params: List):
+        assert launch_result.returncode == int(test_handle.startswith("divergent")), f"{test_handle} return code == {launch_result.returncode}"
+        assert launch_result.stderr == "", f"{test_handle} non-empty stderr {launch_result.stderr}"
+        assert launch_result.stdout == "", f"{test_handle} non-empty stdout {launch_result.stdout}"

regtests/tool_tester/tool_tester/_selfexe.py

@@ -35,6 +35,8 @@ class UnidasmTests(SelfExeTests):
        (otherwise you get a diff error later on).
     3. now run make tests or python $(modulepath)/test_tools.py and check if the new test gets captured. 
 
+    Args:
+        SelfExeTests ([type]): [description]
     """
 
     def __init__(self, assets_path: str):
@@ -45,7 +47,7 @@ def __init__(self, assets_path: str):
         logging.debug(self._bin_test_folder)
 
     def _collect_tests(self):
-        __static_tests = zip( os.listdir(self._asm_test_folder), os.listdir(self._bin_test_folder))
+        __static_tests = zip(os.listdir(self._asm_test_folder), os.listdir(self._bin_test_folder))
         return {
             __asm.split(".")[0]: UnidasmMapper(asm_filename=__asm, bin_filename=__bin) for __asm, __bin in __static_tests
         }