Run CPU-only unit tests automatically for pull requests (#873)

This pull request adds an automation to automatically run unit tests for new pull requests. It checks them against Python 3.8 and 3.11. I updated the supported Python versions to be aligned with those supported in PyTriton, Triton CLI, and GenAI-Perf, especially since Ubuntu 22.04 only supports 3.9+.

The automation uses unittest-parallel to speed up the running of unit tests. I also added the mypy cache to gitignore to avoid accidentally committing those files.

You can see the results in the new jobs that are running for this PR.

Author: dyastremsky

.github/workflows/python-package.yaml

@@ -0,0 +1,59 @@
+# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# This workflow will install Python dependencies and run unit tests.
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-22.04"]
+        python-version: ["3.8", "3.11"]
+    env:
+      SKIP_GPU_TESTS: 1
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install -e .
+    - name: Test with unittest
+      run: |
+        pip install unittest-parallel
+        python3 -m unittest_parallel -v -s ./tests -t .

.gitignore

@@ -366,3 +366,6 @@ MigrationBackup/
 
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
+
+# Files generated by linters
+.mypy_cache/

requirements.txt

@@ -18,6 +18,7 @@ docker>=4.3.1
 gevent>=22.08.0
 grpcio>=1.41.0
 httplib2>=0.19.0
+importlib_metadata>=7.1.0
 matplotlib>=3.3.4
 numba>=0.51.2
 pdfkit>=0.6.1

setup.py

@@ -102,9 +102,10 @@ def get_tag(self):
         "Topic :: Utilities",
         "License :: OSI Approved :: BSD License",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "Environment :: Console",
         "Natural Language :: English",
         "Operating System :: POSIX :: Linux",

tests/test_model_config.py

@@ -262,6 +262,8 @@ def test_write_config_to_file_with_relative_path(self, mock_os_symlink, *args):
 
     def test_instance_group_string(self):
         """Test out all corner cases of instance_group_string()"""
+        if os.getenv("SKIP_GPU_TESTS"):
+            self.skipTest("Skipping this test as it requires GPU")
 
         def _test_helper(config_dict, expected_result, gpu_count=None):
             model_config = ModelConfig.create_from_dictionary(config_dict)
@@ -318,6 +320,8 @@ def _test_helper(config_dict, expected_result, gpu_count=None):
 
     def test_instance_group_count(self):
         """Test out all corner cases of instance_group_count()"""
+        if os.getenv("SKIP_GPU_TESTS"):
+            self.skipTest("Skipping this test as it requires GPU")
 
         def _test_helper(config_dict, expected_result, gpu_count=None):
             model_config = ModelConfig.create_from_dictionary(config_dict)

tests/test_parameter_search.py

@@ -285,7 +285,7 @@ def _create_single_model_no_constraints(self):
 
     def _create_single_model_with_constraints(
         self, latency_budget: str
-    ) -> Union[ConfigCommandProfile | ConfigCommandReport]:
+    ) -> Union[ConfigCommandProfile, ConfigCommandReport]:
         args = [
             "model-analyzer",
             "profile",

tests/test_result_table_manager.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import unittest
 from filecmp import cmp
 from shutil import rmtree
@@ -79,6 +80,8 @@ def test_multi_model_csv_against_golden(self):
         Match the csvs against the golden versions in
         tests/common/multi-model-ckpt
         """
+        if os.getenv("SKIP_GPU_TESTS"):
+            self.skipTest("Skipping this test as it requires GPU")
         table_manager = self._create_multi_model_result_table_manager()
 
         table_manager.create_tables()