Add more unit tests to increase coverage

Author: vkarak

unittests/resources/checks_unlisted/xfailures.py

@@ -0,0 +1,48 @@
+import reframe as rfm
+import reframe.core.builtins as builtins
+import reframe.utility.sanity as sn
+
+
+@rfm.simple_test
+@rfm.xfail('xfail sanity', lambda test: (
+    test.phase == 'sanity' and (test.status == 'xfail' or test.status == 'xpass')
+))
+class XfailTest(rfm.RunOnlyRegressionTest):
+    valid_systems = ['*']
+    valid_prog_environs = ['*']
+    executable = 'echo hello && echo perf=10 && echo time=0.1'
+    reference = {
+        '*': {
+            'perf': (10, 0, 0, 'GB/s'),
+            'time': (0.1, 0, 0, 's')
+        }
+    }
+    status = parameter(['pass', 'fail', 'xfail', 'xpass'])
+    phase  = parameter(['sanity', 'performance'])
+
+    @sanity_function
+    def validate(self):
+        if self.phase == 'sanity' and self.status in ('fail', 'xfail'):
+            return sn.assert_found(r'helo', self.stdout)
+        else:
+            return sn.assert_found(r'hello', self.stdout)
+
+    @run_after('init')
+    def set_references(self):
+        if self.phase == 'sanity':
+            return
+
+        if self.status == 'fail':
+            self.reference['*:perf'] = (9, 0, 0, 'GB/s')
+        elif self.status == 'xfail':
+            self.reference['*:perf'] = builtins.xfail('xfail perf', (9, 0, 0, 'GB/s'))
+        elif self.status == 'xpass':
+            self.reference['*:perf'] = builtins.xfail('xfail perf', self.reference['*:perf'])
+
+    @performance_function('GB/s')
+    def perf(self):
+        return sn.extractsingle(r'perf=(\S+)', self.stdout, 1, float)
+
+    @performance_function('s')
+    def time(self):
+        return sn.extractsingle(r'time=(\S+)', self.stdout, 1, float)

unittests/test_policies.py

@@ -743,3 +743,17 @@ def assert_git_timeout(self):
     runner.runall(testcases)
     assert runner.stats.num_cases() == 2
     assert not runner.stats.failed()
+
+
+def test_expected_failures(make_runner, make_loader, make_exec_ctx):
+    make_exec_ctx()
+    runner = make_runner()
+    loader = make_loader(['unittests/resources/checks_unlisted/xfailures.py'])
+    testcases = executors.generate_testcases(loader.load_all())
+    runner.runall(testcases)
+    assert runner.stats.num_cases() == 8
+
+    # XPASSes are counted also as failures, thuse we expect 4 here
+    assert len(runner.stats.failed()) == 4
+    assert len(runner.stats.xfailed()) == 2
+    assert len(runner.stats.xpassed()) == 2