Add flaky test mark

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Check docs generation with LDoc.
 - Add `--repeat-group` (`-R`) option to run tests in a circle within the group.
+- Add `flaky` test mark.
 
 ## 0.5.7
 

README.rst

@@ -242,6 +242,8 @@ List of luatest functions
 +--------------------------------------------------------------------+-----------------------------------------------+
 | ``xfail_if (condition, message)``                                  | Mark test as xfail if condition is met.       |
 +--------------------------------------------------------------------+-----------------------------------------------+
+| ``flaky (message)``                                                | Mark test as flaky.                           |
++--------------------------------------------------------------------+-----------------------------------------------+
 | ``skip (message)``                                                 | Skip a running test.                          |
 +--------------------------------------------------------------------+-----------------------------------------------+
 | ``skip_if (condition, message)``                                   | Skip a running test if condition is met.      |
@@ -283,6 +285,22 @@ keep xfail tests in sync with an issue tracker.
 XFail only applies to the errors raised by the luatest assertions. Regular Lua
 errors still cause the test failure.
 
+.. _xfail:
+
+---------------------------------
+Flaky
+---------------------------------
+
+The ``flaky`` mark can be used to temporarily remove flaky error from your tests.
+
+.. code-block:: Lua
+
+    local g = t.group()
+    g.test_fail = function()
+        t.flaky("Can be failed and I don't know why ")
+        t.assert_equals(1, math.random(0,1))
+    end
+
 .. _capturing-output:
 
 ---------------------------------

luatest/assertions.lua

@@ -16,6 +16,7 @@ local prettystr_pairs = pp.tostring_pair
 local M = {}
 
 local xfail = false
+local flaky = false
 
 -- private exported functions (for testing)
 M.private = {}
@@ -26,6 +27,13 @@ function M.private.is_xfail()
     return xfail_status
 end
 
+
+function M.private.is_flaky()
+    local flaky_status = flaky
+    flaky = false
+    return flaky_status
+end
+
 --[[--
 
 EPS is meant to help with Lua's floating point math in simple corner
@@ -169,6 +177,14 @@ function M.xfail_if(condition, message)
     end
 end
 
+--- Mark a test as xfail if condition is met
+--
+-- @param condition
+-- @string message
+function M.flaky(message)
+    flaky = message or true
+end
+
 --- Check that two values are equal.
 -- Tables are compared by value.
 --

luatest/output/generic.lua

@@ -50,7 +50,7 @@ end
 -- luacheck: pop
 
 function Output.mt:status_line(colors)
-    colors = colors or {success = '', failure = '', reset = '', xfail = ''}
+    colors = colors or {success = '', failure = '', reset = '', xfail = '', flaky = ''}
     -- return status line string according to results
     local tests = self.result.tests
     local s = {
@@ -69,7 +69,10 @@ function Output.mt:status_line(colors)
     if #tests.error > 0 then
         table.insert(s, string.format("%s%d %s%s", colors.failure, #tests.error, 'errored', colors.reset))
     end
-    if #tests.fail == 0 and #tests.error == 0 and #tests.xsuccess == 0 then
+    if #tests.flaky > 0 then
+        table.insert(s, string.format("%s%d %s%s", colors.flaky, #tests.flaky, 'flaky', colors.reset))
+    end
+    if #tests.fail == 0 and #tests.error == 0 and #tests.xsuccess == 0 and #tests.flaky == 0 then
         table.insert(s, '0 failed')
     end
     if #tests.skip > 0 then

luatest/output/text.lua

@@ -113,6 +113,7 @@ function Output.mt:end_suite()
         failure = self.class.ERROR_COLOR_CODE,
         reset = self.class.RESET_TERM,
         xfail = self.class.WARN_COLOR_CODE,
+        flaky = self.class.WARN_COLOR_CODE,
     }))
     if self.result.notSuccessCount == 0 then
         print('OK')

luatest/runner.lua

@@ -299,6 +299,7 @@ function Runner.mt:start_suite(selected_count, not_selected_count)
             skip = {},
             xfail = {},
             xsuccess = {},
+            flaky = {},
         },
     }
     self.output.result = self.result
@@ -324,7 +325,7 @@ function Runner.mt:update_status(node, err)
     elseif not node:is('success') then
         return
     elseif err.status == 'fail' or err.status == 'error' or err.status == 'skip'
-        or err.status == 'xfail' or err.status == 'xsuccess' then
+        or err.status == 'xfail' or err.status == 'xsuccess' or err.status == 'flaky' then
         node:update_status(err.status, err.message, err.trace)
     else
         error('No such status: ' .. pp.tostring(err.status))
@@ -340,7 +341,7 @@ function Runner.mt:end_test(node)
     if node:is('error') or node:is('fail') or node:is('xsuccess') then
         self.result.aborted = self.fail_fast
     elseif not node:is('success') and not node:is('skip')
-        and not node:is('xfail') then
+        and not node:is('xfail') and not node:is('flaky') then
         error('No such node status: ' .. pp.tostring(node.status))
     end
 end
@@ -377,6 +378,7 @@ function Runner.mt:protected_call(instance, method, pretty_name)
 
     -- check if test was marked as xfail and reset xfail flag
     local xfail = assertions.private.is_xfail()
+    local flaky = assertions.private.is_flaky()
 
     if type(err.message) ~= 'string' then
         err.message = pp.tostring(err.message)
@@ -399,6 +401,11 @@ function Runner.mt:protected_call(instance, method, pretty_name)
         return err
     end
 
+    if flaky and err.status ~= 'error' and err.status ~= 'skip' then
+        err.status = 'flaky'
+        return err
+    end
+
     -- reformat / improve the stack trace
     if pretty_name then -- we do have the real method name
         err.trace = err.trace:gsub("in (%a+) 'method'", "in %1 '" .. pretty_name .. "'")

test/flaky_test.lua

@@ -0,0 +1,40 @@
+local t = require('luatest')
+local g = t.group()
+
+local helper = require('test.helper')
+
+g.test_failed = function()
+    local result = helper.run_suite(function(lu2)
+        local pg = lu2.group('flaky')
+        pg.test_fail = function()
+            lu2.flaky()
+            lu2.assert_equals(2 + 2, 5)
+        end
+    end)
+
+    t.assert_equals(result, 0)
+end
+
+g.test_succeeded = function()
+    local result = helper.run_suite(function(lu2)
+        local pg = lu2.group('flaky')
+        pg.test_success = function()
+            lu2.flaky()
+            lu2.assert_equals(2 + 3, 5)
+        end
+    end)
+
+    t.assert_equals(result, 0)
+end
+
+g.test_error = function()
+    local result = helper.run_suite(function(lu2)
+        local pg = lu2.group('flaky')
+        pg.test_error = function()
+            lu2.flaky()
+            error('Boom!')
+        end
+    end)
+
+    t.assert_equals(result, 1)
+end