Merge pull request #420 from ChrisCummins/ci-fixes

Test fixes and improvements

Author: Chris Cummins

Makefile

@@ -38,7 +38,8 @@ Post-installation Tests
         The same as `make install-test`, but with python test coverage
         reporting. A summary of test coverage is printed at the end of execution
         and the full details are recorded in a coverage.xml file in the project
-        root directory.
+        root directory. To print a report of file coverage to stdout at the end
+        of testing, use argument `PYTEST_ARGS="--cov-report=term"`.
 
     make install-fuzz
         Run the fuzz testing suite against an installed CompilerGym package.
@@ -308,7 +309,7 @@ install-test: install-test-setup
 # environement. This is to ensure that the reported coverage matches that of
 # the value on: https://codecov.io/gh/facebookresearch/CompilerGym
 install-test-cov: install-test-setup
-	export CI=1; $(call pytest,--no-success-flaky-report --benchmark-disable -n auto -k "not fuzz" --durations=5 --cov=compiler_gym --cov-report=xml:$(COV_REPORT) --cov-report=term)
+	export CI=1; $(call pytest,--no-success-flaky-report --benchmark-disable -n auto -k "not fuzz" --durations=5 --cov=compiler_gym --cov-report=xml:$(COV_REPORT))
 
 # The minimum number of seconds to run the fuzz tests in a loop for. Override
 # this at the commandline, e.g. `FUZZ_SECONDS=1800 make fuzz`.

compiler_gym/envs/compiler_env.py

@@ -895,7 +895,8 @@ def raw_step(
                 self.close()
             except ServiceError as e:
                 # close() can raise ServiceError if the service exists with a
-                # non-zero return code. If so,
+                # non-zero return code. We swallow the error here but propagate
+                # the diagnostic message.
                 info[
                     "error_details"
                 ] += f". Additional error during environment closing: {e}"

tests/bin/BUILD

@@ -16,7 +16,6 @@ py_test(
 
 py_test(
     name = "manual_env_bin_test",
-    timeout = "short",
     srcs = ["manual_env_bin_test.py"],
     flaky = 1,
     deps = [

tests/llvm/BUILD

@@ -118,8 +118,9 @@ py_test(
 
 py_test(
     name = "fresh_environment_observation_reward_test",
+    timeout = "long",
     srcs = ["fresh_environment_observation_reward_test.py"],
-    shard_count = 8,
+    shard_count = 12,
     deps = [
         "//compiler_gym/envs",
         "//tests:test_main",
@@ -137,16 +138,6 @@ py_test(
     ],
 )
 
-py_test(
-    name = "gvn_sink_test",
-    srcs = ["gvn_sink_test.py"],
-    deps = [
-        "//compiler_gym/envs",
-        "//tests:test_main",
-        "//tests/pytest_plugins:llvm",
-    ],
-)
-
 py_test(
     name = "gym_interface_compatability",
     timeout = "short",
@@ -217,7 +208,6 @@ py_test(
 
 py_test(
     name = "multiprocessing_test",
-    timeout = "short",
     srcs = ["multiprocessing_test.py"],
     flaky = 1,
     deps = [

tests/llvm/all_actions_single_step_test.py

@@ -7,6 +7,7 @@
 import numpy as np
 
 from compiler_gym.envs import CompilerEnv
+from compiler_gym.service.connection import ServiceError
 from compiler_gym.third_party.autophase import AUTOPHASE_FEATURE_DIM
 from tests.test_main import main
 
@@ -25,6 +26,13 @@ def test_step(env: CompilerEnv, action_name: str):
     assert isinstance(reward, float)
     assert isinstance(done, bool)
 
+    try:
+        env.close()
+    except ServiceError as e:
+        # env.close() will raise an error if the service terminated
+        # ungracefully. In that case, the "done" flag should have been set.
+        assert done, f"Service error was raised when 'done' flag not set: {e}"
+
 
 if __name__ == "__main__":
     main()

tests/llvm/fresh_environment_observation_reward_test.py

@@ -4,12 +4,15 @@
 # LICENSE file in the root directory of this source tree.
 """Integrations tests for the LLVM CompilerGym environments."""
 
+from flaky import flaky
+
 from compiler_gym.envs import CompilerEnv
 from tests.test_main import main
 
 pytest_plugins = ["tests.pytest_plugins.llvm"]
 
 
+@flaky  # Runtime can timeout
 def test_step(env: CompilerEnv, observation_space: str, reward_space: str):
     """Request every combination of observation and reward in a fresh environment."""
     env.reward_space = None

tests/llvm/gvn_sink_test.py

@@ -9,6 +9,7 @@
 
 import gym
 import pytest
+from flaky import flaky
 
 from compiler_gym.envs import LlvmEnv
 from tests.pytest_plugins.common import macos_only
@@ -37,6 +38,7 @@ def process_worker_with_env(env: LlvmEnv, actions: List[int], queue: mp.Queue):
     queue.put((env, observation, reward, done, info))
 
 
+@flaky  # Test contains timeouts.
 def test_running_environment_in_background_process():
     """Test launching and running an LLVM environment in a background process."""
     queue = mp.Queue(maxsize=3)
@@ -46,8 +48,8 @@ def test_running_environment_in_background_process():
     )
     process.start()
     try:
-        process.join(timeout=10)
-        result = queue.get(timeout=10)
+        process.join(timeout=60)
+        result = queue.get(timeout=60)
         observation, reward, done, info = result
 
         assert not done

tests/llvm/multiprocessing_test.py

@@ -10,6 +10,7 @@
 import networkx as nx
 import numpy as np
 import pytest
+from flaky import flaky
 from gym.spaces import Box
 from gym.spaces import Dict as DictSpace
 
@@ -1165,6 +1166,7 @@ def test_object_text_size_observation_spaces(env: LlvmEnv):
     assert value == crc32_code_sizes[sys.platform][2]
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")
     key = "Runtime"
@@ -1188,6 +1190,7 @@ def test_runtime_observation_space(env: LlvmEnv):
     assert len(set(value)) > 1
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     """Test setting a custom observation count for LLVM runtimes."""
     env.reset("cbench-v1/crc32")
@@ -1208,6 +1211,7 @@ def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     assert value.shape == (5,)
 
 
+@flaky  # Runtimes can timeout
 def test_runtime_observation_space_invalid_observation_count(env: LlvmEnv):
     """Test setting an invalid custom observation count for LLVM runtimes."""
     env.reset("cbench-v1/crc32")
@@ -1233,6 +1237,7 @@ def test_runtime_observation_space_not_runnable(env: LlvmEnv):
     assert space.space.contains(value)
 
 
+@flaky  # Build can timeout
 def test_buildtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")
     key = "Buildtime"