Add --repeat testing mode

nikic · nikic · commit 1b3b430f478c · 2020-10-30T17:29:33.000+01:00
This testing mode executes the test multiple times in the same process (but in different requests). It is primarily intended to catch tracing JIT bugs, but also catches state leaks across requests. Closes GH-6365.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -103,3 +103,8 @@ jobs:
         configurationName: DEBUG_NTS_FILE_CACHE
         configurationParameters: '--enable-debug --disable-zts'
         timeoutInMinutes: 90
+    - template: azure/job.yml
+      parameters:
+        configurationName: DEBUG_NTS_REPEAT
+        configurationParameters: '--enable-debug --disable-zts'
+        runTestsParameters: '--repeat 2'
diff --git a/ext/opcache/tests/bug65915.phpt b/ext/opcache/tests/bug65915.phpt
@@ -5,7 +5,11 @@ opcache.enable=1
 opcache.enable_cli=1
 opcache.file_cache_only=0
 --SKIPIF--
-<?php require_once('skipif.inc'); ?>
+<?php
+require_once('skipif.inc');
+// We don't invalidate the file after the second write.
+if (getenv('SKIP_REPEAT')) die("skip Not repeatable");
+?>
 --FILE--
 <?php
 $tmp = __DIR__ . "/bug65915.inc.php";
diff --git a/ext/opcache/tests/zzz_basic_logging.phpt b/ext/opcache/tests/zzz_basic_logging.phpt
@@ -12,7 +12,11 @@ opcache.log_verbosity_level=4
 opcache.huge_code_pages=0
 opcache.preload=
 --SKIPIF--
-<?php require_once('skipif.inc'); ?>
+<?php
+require_once('skipif.inc');
+// Prints "Debug Restarting!" message on next request.
+if (getenv('SKIP_REPEAT')) die("skip Not repeatable");
+?>
 --FILE--
 <?php
 echo "Foo Bar\n";
diff --git a/ext/pcntl/tests/002.phpt b/ext/pcntl/tests/002.phpt
@@ -7,6 +7,7 @@ pcntl: pcntl_sigprocmask(), pcntl_sigwaitinfo(), pcntl_sigtimedwait()
     elseif (!function_exists('pcntl_sigwaitinfo') or !function_exists('pcntl_sigtimedwait')) die('skip required functionality is not available');
     elseif (!defined('CLD_EXITED')) die('skip CLD_EXITED not defined');
     elseif (getenv('SKIP_ASAN')) die('skip Fails intermittently under asan/msan');
+    elseif (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_01.phpt b/ext/pcntl/tests/pcntl_unshare_01.phpt
@@ -9,6 +9,7 @@ if (!defined("CLONE_NEWUSER")) die("skip flag unavailable");
 if (@pcntl_unshare(CLONE_NEWUSER) == false && pcntl_get_last_error() == PCNTL_EPERM) {
     die("skip Insufficient privileges to use CLONE_NEWUSER");
 }
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_02.phpt b/ext/pcntl/tests/pcntl_unshare_02.phpt
@@ -15,6 +15,7 @@ if (posix_getuid() !== 0 &&
 if (@pcntl_unshare(CLONE_NEWPID) == false && pcntl_get_last_error() == PCNTL_EPERM) {
     die("skip Insufficient privileges for CLONE_NEWPID");
 }
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pcntl/tests/pcntl_unshare_03.phpt b/ext/pcntl/tests/pcntl_unshare_03.phpt
@@ -15,6 +15,7 @@ if (@pcntl_unshare(CLONE_NEWNET) == false && pcntl_get_last_error() == PCNTL_EPE
     die("skip Insufficient privileges for CLONE_NEWPID");
 }
 if (getenv("SKIP_ONLINE_TESTS")) die("skip online test");
+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");
 ?>
 --FILE--
 <?php
diff --git a/ext/pgsql/tests/skipif.inc b/ext/pgsql/tests/skipif.inc
@@ -12,6 +12,11 @@ include("lcmess.inc");
 if (!extension_loaded("pgsql")) {
     die("skip pgsql extension not loaded\n");
 }
+if (getenv("SKIP_REPEAT")) {
+    // pgsql tests are order-dependent.
+    // We should probably change that, but in the meantime do not allow repetition.
+    die("skip Cannot repeat pgsql tests");
+}
 $conn = @pg_connect($conn_str);
 if (!is_resource($conn)) {
     die("skip could not connect\n");
diff --git a/ext/readline/tests/libedit_info_001.phpt b/ext/readline/tests/libedit_info_001.phpt
@@ -6,6 +6,7 @@ if (READLINE_LIB != "libedit") die("skip libedit only");
 if(substr(PHP_OS, 0, 3) == 'WIN' ) {
     die('skip not for windows');
 }
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/libedit_write_history_001.phpt b/ext/readline/tests/libedit_write_history_001.phpt
@@ -6,6 +6,7 @@ if (READLINE_LIB != "libedit") die("skip libedit only");
 if(substr(PHP_OS, 0, 3) == 'WIN' ) {
     die('skip not for windows');
 }
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/readline_info_001.phpt b/ext/readline/tests/readline_info_001.phpt
@@ -3,6 +3,7 @@ readline_info(): Basic test
 --SKIPIF--
 <?php if (!extension_loaded("readline")) die("skip");
 if (READLINE_LIB == "libedit") die("skip readline only");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/readline/tests/readline_read_history_001.phpt b/ext/readline/tests/readline_read_history_001.phpt
@@ -1,7 +1,10 @@
 --TEST--
 readline_read_history(): Basic test
 --SKIPIF--
-<?php if (!extension_loaded("readline") || !function_exists('readline_list_history')) die("skip"); ?>
+<?php
+if (!extension_loaded("readline") || !function_exists('readline_list_history')) die("skip");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
+?>
 --FILE--
 <?php
 
diff --git a/ext/readline/tests/readline_write_history_001.phpt b/ext/readline/tests/readline_write_history_001.phpt
@@ -3,6 +3,7 @@ readline_write_history(): Basic test
 --SKIPIF--
 <?php if (!extension_loaded("readline") || !function_exists('readline_add_history')) die("skip");
 if (READLINE_LIB == "libedit") die("skip readline only");
+if (getenv('SKIP_REPEAT')) die("skip readline has global state");
 ?>
 --FILE--
 <?php
diff --git a/ext/standard/tests/file/realpath_bug77484.phpt b/ext/standard/tests/file/realpath_bug77484.phpt
@@ -9,6 +9,10 @@ if (PHP_ZTS) {
     /* TODO eliminate difference in TS build. */
     die("skip Not for ZTS");
 }
+if (getenv('SKIP_REPEAT')) {
+    /* The cwd is persistent across repeats */
+    die("skip Not repeatable");
+}
 ?>
 --FILE--
 <?php
diff --git a/ext/standard/tests/general_functions/http_response_code.phpt b/ext/standard/tests/general_functions/http_response_code.phpt
@@ -2,6 +2,15 @@
 Test http_response_code basic functionality
 --CREDITS--
 Gabriel Caruso (carusogabriel@php.net)
+--SKIPIF--
+<?php
+if (getenv('SKIP_REPEAT')) {
+    /* The http_response_code leaks across repeats. Technically that's a bug, but it's something
+     * that only affects the CLI repeat option, where the response code is not meaningful in the
+     * first place. Other SAPIs will properly reset the response code for each request. */
+    die('skip Not repeatable');
+}
+?>
 --FILE--
 <?php
 var_dump(
diff --git a/ext/standard/tests/general_functions/proc_nice_basic.phpt b/ext/standard/tests/general_functions/proc_nice_basic.phpt
@@ -25,7 +25,7 @@ if ($exit_code !== 0) {
         else
             return -1;
     }
-    $delta = 10;
+    $delta = 5;
     $pid = getmypid();
     $niceBefore = getNice($pid);
     proc_nice($delta);
diff --git a/run-tests.php b/run-tests.php
@@ -147,7 +147,7 @@ function main(): void
            $repeat, $result_tests_file, $slow_min_ms, $start_time, $switch,
            $temp_source, $temp_target, $test_cnt, $test_dirs,
            $test_files, $test_idx, $test_list, $test_results, $testfile,
-           $user_tests, $valgrind, $sum_results, $shuffle, $file_cache;
+           $user_tests, $valgrind, $sum_results, $shuffle, $file_cache, $num_repeats;
     // Parallel testing
     global $workers, $workerID;
     global $context_line_count;
@@ -405,6 +405,7 @@ function main(): void
     $shuffle = false;
     $workers = null;
     $context_line_count = 3;
+    $num_repeats = 1;
 
     $cfgtypes = ['show', 'keep'];
     $cfgfiles = ['skip', 'php', 'clean', 'out', 'diff', 'exp', 'mem'];
@@ -623,6 +624,10 @@ function main(): void
                             . ':print_suppressions=0';
                     }
                     break;
+                case '--repeat':
+                    $num_repeats = (int) $argv[++$i];
+                    $environment['SKIP_REPEAT'] = 1;
+                    break;
                 //case 'w'
                 case '-':
                     // repeat check with full switch
@@ -1802,6 +1807,13 @@ function show_file_block(string $file, string $block, ?string $section = null):
     }
 }
 
+function skip_test(string $tested, string $tested_file, string $shortname, string $reason) {
+    show_result('SKIP', $tested, $tested_file, "reason: $reason");
+    junit_init_suite(junit_get_suitename_for($shortname));
+    junit_mark_test_as('SKIP', $shortname, $tested, 0, $reason);
+    return 'SKIPPED';
+}
+
 //
 //  Run an individual test case.
 //
@@ -1818,6 +1830,7 @@ function run_test(string $php, $file, array $env): string
     global $no_file_cache;
     global $slow_min_ms;
     global $preload, $file_cache;
+    global $num_repeats;
     // Parallel testing
     global $workerID;
     $temp_filenames = null;
@@ -1911,6 +1924,10 @@ function run_test(string $php, $file, array $env): string
         }
     }
 
+    $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
+    $tested_file = $shortname;
+    $tested = trim($section_text['TEST']);
+
     // the redirect section allows a set of tests to be reused outside of
     // a given test dir
     if ($bork_info === null) {
@@ -1928,6 +1945,10 @@ function run_test(string $php, $file, array $env): string
                 unset($section_text['FILEEOF']);
             }
 
+            if ($num_repeats > 1 && isset($section_text['FILE_EXTERNAL'])) {
+                return skip_test($tested, $tested_file, $shortname, 'Test with FILE_EXTERNAL might not be repeatable');
+            }
+
             foreach (['FILE', 'EXPECT', 'EXPECTF', 'EXPECTREGEX'] as $prefix) {
                 $key = $prefix . '_EXTERNAL';
 
@@ -1951,9 +1972,6 @@ function run_test(string $php, $file, array $env): string
     }
     fclose($fp);
 
-    $shortname = str_replace(TEST_PHP_SRCDIR . '/', '', $file);
-    $tested_file = $shortname;
-
     if ($bork_info !== null) {
         show_result("BORK", $bork_info, $tested_file);
         $PHP_FAILED_TESTS['BORKED'][] = [
@@ -1983,8 +2001,6 @@ function run_test(string $php, $file, array $env): string
         $cmdRedirect = '';
     }
 
-    $tested = trim($section_text['TEST']);
-
     /* For GET/POST/PUT tests, check if cgi sapi is available and if it is, use it. */
     if (array_key_exists('CGI', $section_text) || !empty($section_text['GET']) || !empty($section_text['POST']) || !empty($section_text['GZIP_POST']) || !empty($section_text['DEFLATE_POST']) || !empty($section_text['POST_RAW']) || !empty($section_text['PUT']) || !empty($section_text['COOKIE']) || !empty($section_text['EXPECTHEADERS'])) {
         if (isset($php_cgi)) {
@@ -1999,13 +2015,12 @@ function run_test(string $php, $file, array $env): string
             } elseif (file_exists(dirname($php) . "/php-cgi")) {
                 $php = realpath(dirname($php) . "/php-cgi") . ' -C ';
             } else {
-                show_result('SKIP', $tested, $tested_file, "reason: CGI not available");
-
-                junit_init_suite(junit_get_suitename_for($shortname));
-                junit_mark_test_as('SKIP', $shortname, $tested, 0, 'CGI not available');
-                return 'SKIPPED';
+                return skip_test($tested, $tested_file, $shortname, 'CGI not available');
             }
         }
+        if ($num_repeats > 1) {
+            return skip_test($tested, $tested_file, $shortname, 'CGI does not support --repeat');
+        }
         $uses_cgi = true;
     }
 
@@ -2023,11 +2038,22 @@ function run_test(string $php, $file, array $env): string
             // be run straight away. For example, EXTENSIONS, SKIPIF, CLEAN.
             $extra_options = '-rr';
         } else {
-            show_result('SKIP', $tested, $tested_file, "reason: phpdbg not available");
+            return skip_test($tested, $tested_file, $shortname, 'phpdbg not available');
+        }
+        if ($num_repeats > 1) {
+            return skip_test($tested, $tested_file, $shortname, 'phpdbg does not support --repeat');
+        }
+    }
 
-            junit_init_suite(junit_get_suitename_for($shortname));
-            junit_mark_test_as('SKIP', $shortname, $tested, 0, 'phpdbg not available');
-            return 'SKIPPED';
+    if ($num_repeats > 1) {
+        if (array_key_exists('CLEAN', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with CLEAN might not be repeatable');
+        }
+        if (array_key_exists('STDIN', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with STDIN might not be repeatable');
+        }
+        if (array_key_exists('CAPTURE_STDIO', $section_text)) {
+            return skip_test($tested, $tested_file, $shortname, 'Test with CAPTURE_STDIO might not be repeatable');
         }
     }
 
@@ -2173,6 +2199,9 @@ function run_test(string $php, $file, array $env): string
             // even though all the files are re-created.
             $ini_settings['opcache.validate_timestamps'] = '0';
         }
+    } else if ($num_repeats > 1) {
+        // Make sure warnings still show up on the second run.
+        $ini_settings['opcache.record_warnings'] = '1';
     }
 
     // Any special ini settings
@@ -2183,6 +2212,10 @@ function run_test(string $php, $file, array $env): string
         $replacement = IS_WINDOWS ? '"' . PHP_BINARY . ' -r \"while ($in = fgets(STDIN)) echo $in;\" > $1"' : 'tee $1 >/dev/null';
         $section_text['INI'] = preg_replace('/{MAIL:(\S+)}/', $replacement, $section_text['INI']);
         settings2array(preg_split("/[\n\r]+/", $section_text['INI']), $ini_settings);
+
+        if ($num_repeats > 1 && isset($ini_settings['opcache.opt_debug_level'])) {
+            return skip_test($tested, $tested_file, $shortname, 'opt_debug_level tests are not repeatable');
+        }
     }
 
     $ini_settings = settings2params($ini_settings);
@@ -2484,7 +2517,8 @@ function run_test(string $php, $file, array $env): string
         $env['CONTENT_TYPE'] = '';
         $env['CONTENT_LENGTH'] = '';
 
-        $cmd = "$php $pass_options $ini_settings -f \"$test_file\" $args$cmdRedirect";
+        $repeat_option = $num_repeats > 1 ? "--repeat $num_repeats" : "";
+        $cmd = "$php $pass_options $repeat_option $ini_settings -f \"$test_file\" $args$cmdRedirect";
     }
 
     if ($valgrind) {
@@ -2557,6 +2591,25 @@ function run_test(string $php, $file, array $env): string
         }
     }
 
+    if ($num_repeats > 1) {
+        // In repeat mode, retain the output before the first execution,
+        // and of the last execution. Do this early, because the trimming below
+        // makes the newline handling complicated.
+        $separator1 = "Executing for the first time...\n";
+        $separator1_pos = strpos($out, $separator1);
+        if ($separator1_pos !== false) {
+            $separator2 = "Finished execution, repeating...\n";
+            $separator2_pos = strrpos($out, $separator2);
+            if ($separator2_pos !== false) {
+                $out = substr($out, 0, $separator1_pos)
+                     . substr($out, $separator2_pos + strlen($separator2));
+            } else {
+                $out = substr($out, 0, $separator1_pos)
+                     . substr($out, $separator1_pos + strlen($separator1));
+            }
+        }
+    }
+
     // Does the output match what is expected?
     $output = preg_replace("/\r\n/", "\n", trim($out));
 
diff --git a/sapi/cli/php_cli.c b/sapi/cli/php_cli.c

Original file line number	Diff line number	Diff line change
`@@ -9,6 +9,7 @@ if (!defined("CLONE_NEWUSER")) die("skip flag unavailable");`
`9`	`9`	`if (@pcntl_unshare(CLONE_NEWUSER) == false && pcntl_get_last_error() == PCNTL_EPERM) {`
`10`	`10`	`die("skip Insufficient privileges to use CLONE_NEWUSER");`
`11`	`11`	`}`
	`12`	`+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");`
`12`	`13`	`?>`
`13`	`14`	`--FILE--`
`14`	`15`	`<?php`
Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ if (posix_getuid() !== 0 &&`
`15`	`15`	`if (@pcntl_unshare(CLONE_NEWPID) == false && pcntl_get_last_error() == PCNTL_EPERM) {`
`16`	`16`	`die("skip Insufficient privileges for CLONE_NEWPID");`
`17`	`17`	`}`
	`18`	`+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");`
`18`	`19`	`?>`
`19`	`20`	`--FILE--`
`20`	`21`	`<?php`
Original file line number	Diff line number	Diff line change
`@@ -15,6 +15,7 @@ if (@pcntl_unshare(CLONE_NEWNET) == false && pcntl_get_last_error() == PCNTL_EPE`
`15`	`15`	`die("skip Insufficient privileges for CLONE_NEWPID");`
`16`	`16`	`}`
`17`	`17`	`if (getenv("SKIP_ONLINE_TESTS")) die("skip online test");`
	`18`	`+if (getenv("SKIP_REPEAT")) die("skip cannot be repeated");`
`18`	`19`	`?>`
`19`	`20`	`--FILE--`
`20`	`21`	`<?php`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,7 @@ if (READLINE_LIB != "libedit") die("skip libedit only");`
`6`	`6`	`if(substr(PHP_OS, 0, 3) == 'WIN' ) {`
`7`	`7`	`die('skip not for windows');`
`8`	`8`	`}`
	`9`	`+if (getenv('SKIP_REPEAT')) die("skip readline has global state");`
`9`	`10`	`?>`
`10`	`11`	`--FILE--`
`11`	`12`	`<?php`