Convert stderr and stdout in status to strings on assignment (Khronos…

…Group#3049)

This avoids Python2 vs Python3 issues related to how we decode bytes
later on in the tests.

Switching over to using unittest instead of nosetest

CMakeLists.txt

@@ -29,7 +29,6 @@ enable_testing()
 set(SPIRV_TOOLS "SPIRV-Tools")
 
 include(GNUInstallDirs)
-include(cmake/setup_build.cmake)
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 set(CMAKE_CXX_STANDARD 11)

test/tools/CMakeLists.txt

@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-spirv_add_nosetests(expect)
-spirv_add_nosetests(spirv_test_framework)
-
+add_test(NAME spirv-tools_expect_unittests
+         COMMAND ${PYTHON_EXECUTABLE} -m unittest expect_unittest.py
+         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
+add_test(NAME spirv-tools_spirv_test_framework_unittests
+         COMMAND ${PYTHON_EXECUTABLE} -m unittest spirv_test_framework_unittest.py
+         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
 add_subdirectory(opt)

test/tools/expect.py

@@ -510,7 +510,7 @@ def check_has_error_message_as_substring(self, status):
     if not status.stderr:
       return False, 'Expected error message, but no output on stderr'
     if self.expected_error_substr not in convert_to_unix_line_endings(
-        status.stderr.decode('utf8')):
+        status.stderr):
       return False, ('Incorrect stderr output:\n{act}\n'
                      'Expected substring not found in stderr:\n{exp}'.format(
                          act=status.stderr, exp=self.expected_error_substr))
@@ -530,7 +530,7 @@ def check_has_warning_message(self, status):
                      ' command execution')
     if not status.stderr:
       return False, 'Expected warning message, but no output on stderr'
-    if self.expected_warning != convert_to_unix_line_endings(status.stderr.decode('utf8')):
+    if self.expected_warning != convert_to_unix_line_endings(status.stderr):
       return False, ('Incorrect stderr output:\n{act}\n'
                      'Expected:\n{exp}'.format(
                          act=status.stderr, exp=self.expected_warning))
@@ -590,16 +590,16 @@ def check_stdout_match(self, status):
       if not status.stdout:
         return False, 'Expected something on stdout'
     elif type(self.expected_stdout) == str:
-      if self.expected_stdout != convert_to_unix_line_endings(status.stdout.decode('utf8')):
+      if self.expected_stdout != convert_to_unix_line_endings(status.stdout):
         return False, ('Incorrect stdout output:\n{ac}\n'
                        'Expected:\n{ex}'.format(
                            ac=status.stdout, ex=self.expected_stdout))
     else:
-      converted = convert_to_unix_line_endings(status.stdout.decode('utf8'))
+      converted = convert_to_unix_line_endings(status.stdout)
       if not self.expected_stdout.search(converted):
         return False, ('Incorrect stdout output:\n{ac}\n'
                        'Expected to match regex:\n{ex}'.format(
-                           ac=status.stdout.decode('utf8'), ex=self.expected_stdout.pattern))
+                           ac=status.stdout, ex=self.expected_stdout.pattern))
     return True, ''
 
 
@@ -624,13 +624,13 @@ def check_stderr_match(self, status):
       if not status.stderr:
         return False, 'Expected something on stderr'
     elif type(self.expected_stderr) == str:
-      if self.expected_stderr != convert_to_unix_line_endings(status.stderr.decode('utf8')):
+      if self.expected_stderr != convert_to_unix_line_endings(status.stderr):
         return False, ('Incorrect stderr output:\n{ac}\n'
                        'Expected:\n{ex}'.format(
                            ac=status.stderr, ex=self.expected_stderr))
     else:
       if not self.expected_stderr.search(
-          convert_to_unix_line_endings(status.stderr.decode('utf8'))):
+          convert_to_unix_line_endings(status.stderr)):
         return False, ('Incorrect stderr output:\n{ac}\n'
                        'Expected to match regex:\n{ex}'.format(
                            ac=status.stderr, ex=self.expected_stderr.pattern))
@@ -695,7 +695,7 @@ def check_list_of_executed_passes(self, status):
     # Collect all the output lines containing a pass name.
     pass_names = []
     pass_name_re = re.compile(r'.*IR before pass (?P<pass_name>[\S]+)')
-    for line in status.stderr.decode('utf8').splitlines():
+    for line in status.stderr.splitlines():
       match = pass_name_re.match(line)
       if match:
         pass_names.append(match.group('pass_name'))

test/tools/expect_unittest.py

@@ -0,0 +1,82 @@
+# Copyright (c) 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for the expect module."""
+
+import expect
+from spirv_test_framework import TestStatus
+import re
+import unittest
+
+
+class TestStdoutMatchADotC(expect.StdoutMatch):
+    expected_stdout = re.compile('a.c')
+
+
+class TestExpect(unittest.TestCase):
+    def test_get_object_name(self):
+        """Tests get_object_filename()."""
+        source_and_object_names = [('a.vert', 'a.vert.spv'),
+                                   ('b.frag', 'b.frag.spv'),
+                                   ('c.tesc', 'c.tesc.spv'),
+                                   ('d.tese', 'd.tese.spv'),
+                                   ('e.geom', 'e.geom.spv'),
+                                   ('f.comp', 'f.comp.spv'),
+                                   ('file', 'file.spv'), ('file.', 'file.spv'),
+                                   ('file.uk',
+                                    'file.spv'), ('file.vert.',
+                                                  'file.vert.spv'),
+                                   ('file.vert.bla',
+                                    'file.vert.spv')]
+        actual_object_names = [
+            expect.get_object_filename(f[0]) for f in source_and_object_names
+        ]
+        expected_object_names = [f[1] for f in source_and_object_names]
+
+        self.assertEqual(actual_object_names, expected_object_names)
+
+    def test_stdout_match_regex_has_match(self):
+        test = TestStdoutMatchADotC()
+        status = TestStatus(
+            test_manager=None,
+            returncode=0,
+            stdout=b'0abc1',
+            stderr=None,
+            directory=None,
+            inputs=None,
+            input_filenames=None)
+        self.assertTrue(test.check_stdout_match(status)[0])
+
+    def test_stdout_match_regex_no_match(self):
+        test = TestStdoutMatchADotC()
+        status = TestStatus(
+            test_manager=None,
+            returncode=0,
+            stdout=b'ab',
+            stderr=None,
+            directory=None,
+            inputs=None,
+            input_filenames=None)
+        self.assertFalse(test.check_stdout_match(status)[0])
+
+    def test_stdout_match_regex_empty_stdout(self):
+        test = TestStdoutMatchADotC()
+        status = TestStatus(
+            test_manager=None,
+            returncode=0,
+            stdout=b'',
+            stderr=None,
+            directory=None,
+            inputs=None,
+            input_filenames=None)
+        self.assertFalse(test.check_stdout_match(status)[0])

test/tools/spirv_test_framework.py

@@ -70,7 +70,7 @@ def get_all_methods(instance):
 
 
 def get_all_superclasses(cls):
-  """Returns all superclasses of a given class.
+  """Returns all superclasses of a given class. Omits root 'object' superclass.
 
     Returns:
       A list of superclasses of the given class. The order guarantees that
@@ -83,11 +83,12 @@ def get_all_superclasses(cls):
   classes = []
   for superclass in cls.__bases__:
     for c in get_all_superclasses(superclass):
-      if c not in classes:
+      if c is not object and c not in classes:
         classes.append(c)
   for superclass in cls.__bases__:
-    if superclass not in classes:
+    if superclass is not object and superclass not in classes:
       classes.append(superclass)
+
   return classes
 
 
@@ -142,8 +143,28 @@ def __init__(self, test_manager, returncode, stdout, stderr, directory,
                inputs, input_filenames):
     self.test_manager = test_manager
     self.returncode = returncode
-    self.stdout = stdout
-    self.stderr = stderr
+    # Some of our MacOS bots still run Python 2, so need to be backwards
+    # compatible here.
+    if type(stdout) is not str:     
+      if sys.version_info[0] is 2:
+       self.stdout = stdout.decode('utf-8')
+      elif sys.version_info[0] is 3:
+        self.stdout = str(stdout, encoding='utf-8') if stdout is not None else stdout
+      else:
+        raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
+    else:
+      self.stdout = stdout
+
+    if type(stderr) is not str:     
+      if sys.version_info[0] is 2:
+       self.stderr = stderr.decode('utf-8')
+      elif sys.version_info[0] is 3:
+        self.stderr = str(stderr, encoding='utf-8') if stderr is not None else stderr
+      else:
+        raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
+    else:
+      self.stderr = stderr
+
     # temporary directory where the test runs
     self.directory = directory
     # List of inputs, as PlaceHolder objects.