TSPS-423 add smoke test for teaspoons (#191)

Co-authored-by: Jose Soto <[email protected]>

smoke-test/README.md

@@ -0,0 +1,58 @@
+# Teaspoons Smoke Tests
+
+These smoke tests provide a means for running a small set of tests against a live running Teaspoons instance to validate that
+it is up and functional.  These tests should verify more than the `/status` endpoint and should additionally try to 
+verify some basic functionality of Teaspoons.  
+
+These tests should run quickly (no longer than a few seconds), should be idempotent, and when possible, should not 
+make any changes to the state of the service or its data.  
+
+## Requirements
+
+Python 3.10.3 or higher
+
+## Setup
+
+You will need to install required pip libraries:
+
+```pip install -r requirements.txt```
+
+## Run
+
+The smoke tests have 2 different modes that they can run in: authenticated or unauthenticated.  The mode will be 
+automatically selected based on the arguments you pass to `smoke_test.py`.
+
+To run the _unauthenticated_ smoke tests:
+
+```python smoke_test.py {TEASPOONS_HOST}```
+
+```python smoke_test.py teaspoons.dsde-dev.broadinstitute.org```
+
+To run all (_authenticated_ and _unauthenticated_) smoke tests:
+
+```python smoke_test.py {TEASPOONS_HOST} $(gcloud auth print-access-token)```
+
+```python smoke_test.py teaspoons.dsde-dev.broadinstitute.org $(gcloud auth print-access-token)```
+
+## Required and Optional Arguments
+
+### TEASPOONS_HOST
+Required - Can be just a domain or a domain and port:
+
+* `teaspoons.dsde-dev.broadinstitute.org`
+* `teaspoons.dsde-dev.broadinstitute.org:443`
+
+The protocol can also be added if you desire, however, most Teaspoons instances can and should use HTTPS
+and this is the default if no protocol is specified:
+
+* `https://teaspoons.dsde-dev.broadinstitute.org`
+
+### USER_TOKEN
+Optional - A `gcloud` access token.  If present, `smoke_test.py` will execute all unauthenticated tests as well as all
+authenticated tests using the access token provided in this argument. 
+
+### Verbosity
+Optional - You may control how much information is printed to `STDOUT` while running the smoke tests by passing a 
+verbosity argument to `smoke_test.py`.  For example to print more information about the tests being run:
+
+```python -v 2 smoke_test.py {TEASPOONS_HOST}```

smoke-test/requirements.txt

@@ -0,0 +1 @@
+requests>=2.28.1

smoke-test/smoke_test.py

@@ -0,0 +1,106 @@
+import argparse
+import sys
+import unittest
+from unittest import TestSuite
+
+import requests
+
+from tests.authenticated.teaspoons_jobs_list_tests import TeaspoonsJobsListTests
+from tests.authenticated.teaspoons_pipeline_runs_list_tests import TeaspoonsPipelineRunsListTests
+from tests.authenticated.teaspoons_pipelines_list_tests import TeaspoonsPipelinesListTests
+from tests.teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+from tests.unauthenticated.teaspoons_status_tests import TeaspoonsStatusTests
+from tests.unauthenticated.teaspoons_version_tests import TeaspoonsVersionTests
+
+DESCRIPTION = """
+Teaspoons Smoke Test
+Enter the host (domain and optional port) of the Teaspoons instance you want to to test. 
+This test will ensure that the Teaspoons instance running on that host is minimally functional.
+"""
+
+
+def gather_tests(is_authenticated: bool = False) -> TestSuite:
+    suite = unittest.TestSuite()
+
+    status_tests = unittest.defaultTestLoader.loadTestsFromTestCase(TeaspoonsStatusTests)
+    version_tests = unittest.defaultTestLoader.loadTestsFromTestCase(TeaspoonsVersionTests)
+
+    suite.addTests(status_tests)
+    suite.addTests(version_tests)
+
+    if is_authenticated:
+        pipeline_list_tests = unittest.defaultTestLoader.loadTestsFromTestCase(TeaspoonsPipelinesListTests)
+        pipeline_runs_list_tests = unittest.defaultTestLoader.loadTestsFromTestCase(TeaspoonsPipelineRunsListTests)
+        jobs_list_tests = unittest.defaultTestLoader.loadTestsFromTestCase(TeaspoonsJobsListTests)
+
+        suite.addTests(pipeline_list_tests)
+        suite.addTests(pipeline_runs_list_tests)
+        suite.addTests(jobs_list_tests)
+    else:
+        print("No User Token provided.  Skipping authenticated tests.")
+
+    return suite
+
+
+def main(main_args):
+    if main_args.user_token:
+        verify_user_token(main_args.user_token)
+
+    TeaspoonsSmokeTestCase.TEASPOONS_HOST = main_args.teaspoons_host
+    TeaspoonsSmokeTestCase.USER_TOKEN = main_args.user_token
+
+    test_suite = gather_tests(main_args.user_token)
+
+    runner = unittest.TextTestRunner(verbosity=main_args.verbosity)
+    result = runner.run(test_suite)
+
+    # system exit if any tests fail
+    if result.failures or result.errors:
+        sys.exit(1)
+
+
+def verify_user_token(user_token: str) -> bool:
+    response = requests.get(f"https://www.googleapis.com/oauth2/v1/tokeninfo?access_token={user_token}")
+    assert response.status_code == 200, "User Token is no longer valid.  Please generate a new token and try again."
+
+
+if __name__ == "__main__":
+    try:
+        parser = argparse.ArgumentParser(
+            description=DESCRIPTION,
+            formatter_class=argparse.RawTextHelpFormatter
+        )
+        parser.add_argument(
+            "-v",
+            "--verbosity",
+            type=int,
+            choices=[0, 1, 2],
+            default=1,
+            help="""Python unittest verbosity setting: 
+0: Quiet - Prints only number of tests executed
+1: Minimal - (default) Prints number of tests executed plus a dot for each success and an F for each failure
+2: Verbose - Help string and its result will be printed for each test"""
+        )
+        parser.add_argument(
+            "teaspoons_host",
+            help="domain with optional port number of the Teasponns host you want to test"
+        )
+        parser.add_argument(
+            "user_token",
+            nargs='?',
+            default=None,
+            help="Optional. If present, will test additional authenticated endpoints using the specified token"
+        )
+
+        args = parser.parse_args()
+
+        # Need to pop off sys.argv values to avoid messing with args passed to unittest.main()
+        for _ in range(len(sys.argv[1:])):
+            sys.argv.pop()
+
+        main(args)
+        sys.exit(0)
+
+    except Exception as e:
+        print(e)
+        sys.exit(1)

smoke-test/tests/authenticated/teaspoons_jobs_list_tests.py

@@ -0,0 +1,14 @@
+from ..teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+
+
+class TeaspoonsJobsListTests(TeaspoonsSmokeTestCase):
+    '''
+    Test the jobs list endpoint for a 200 status code
+    '''
+    @staticmethod
+    def jobs_list_url() -> str:
+        return TeaspoonsSmokeTestCase.build_teaspoons_url("api/job/v1/jobs?limit=10")
+
+    def test_status_code_is_200(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.jobs_list_url(), TeaspoonsSmokeTestCase.USER_TOKEN)
+        self.assertEqual(response.status_code, 200, f"Jobs List HTTP Status is not 200: {response.text}")

smoke-test/tests/authenticated/teaspoons_pipeline_runs_list_tests.py

@@ -0,0 +1,14 @@
+from ..teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+
+
+class TeaspoonsPipelineRunsListTests(TeaspoonsSmokeTestCase):
+    '''
+    Test the pipeline runs list endpoint for a 200 status code
+    '''
+    @staticmethod
+    def pipeline_runs_list_url() -> str:
+        return TeaspoonsSmokeTestCase.build_teaspoons_url("api/pipelineruns/v1/pipelineruns?limit=10")
+
+    def test_status_code_is_200(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.pipeline_runs_list_url(), TeaspoonsSmokeTestCase.USER_TOKEN)
+        self.assertEqual(response.status_code, 200, f"Pipeline Runs List HTTP Status is not 200: {response.text}")

smoke-test/tests/authenticated/teaspoons_pipelines_list_tests.py

@@ -0,0 +1,22 @@
+import json
+
+from ..teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+
+
+class TeaspoonsPipelinesListTests(TeaspoonsSmokeTestCase):
+    '''
+    Test the pipeline list endpoint for a 200 status code and that the response has
+    greater than 0 responses
+    '''
+    @staticmethod
+    def pipelines_list_url() -> str:
+        return TeaspoonsSmokeTestCase.build_teaspoons_url("/api/pipelines/v1")
+
+    def test_status_code_is_200(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.pipelines_list_url(), TeaspoonsSmokeTestCase.USER_TOKEN)
+        self.assertEqual(response.status_code, 200, f"Pipelines List HTTP Status is not 200: {response.text}")
+
+    def test_pipelines_list_greater_than_zero(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.pipelines_list_url(), TeaspoonsSmokeTestCase.USER_TOKEN)
+        pipeline_info = json.loads(response.text)
+        self.assertGreater(len(pipeline_info["results"]), 0, "No pipelines found")

smoke-test/tests/teaspoons_smoke_test_case.py

@@ -0,0 +1,31 @@
+import re
+from functools import cache
+from unittest import TestCase
+from urllib.parse import urljoin
+
+import requests
+from requests import Response
+
+
+class TeaspoonsSmokeTestCase(TestCase):
+    '''
+    Base class for all Teaspoons smoke tests. Contains static methods that all tests
+    can use.
+    '''
+    TEASPOONS_HOST = None
+    USER_TOKEN = None
+
+    @staticmethod
+    def build_teaspoons_url(path: str) -> str:
+        assert TeaspoonsSmokeTestCase.TEASPOONS_HOST, "ERROR - TeaspoonsSmokeTests.TEASPOONS_HOST not properly set"
+        if re.match(r"^\s*https?://", TeaspoonsSmokeTestCase.TEASPOONS_HOST):
+            return urljoin(TeaspoonsSmokeTestCase.TEASPOONS_HOST, path)
+        else:
+            return urljoin(f"https://{TeaspoonsSmokeTestCase.TEASPOONS_HOST}", path)
+
+    @staticmethod
+    @cache
+    def call_teaspoons(url: str, user_token: str = None) -> Response:
+        """Function is memoized so that we only make the call once"""
+        headers = {"Authorization": f"Bearer {user_token}"} if user_token else {}
+        return requests.get(url, headers=headers)

smoke-test/tests/unauthenticated/teaspoons_status_tests.py

@@ -0,0 +1,14 @@
+from ..teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+
+
+class TeaspoonsStatusTests(TeaspoonsSmokeTestCase):
+    '''
+    Test the status endpoint for a 200 status code
+    '''
+    @staticmethod
+    def status_url() -> str:
+        return TeaspoonsSmokeTestCase.build_teaspoons_url("/status")
+
+    def test_status_code_is_200(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.status_url())
+        self.assertEqual(response.status_code, 200)

smoke-test/tests/unauthenticated/teaspoons_version_tests.py

@@ -0,0 +1,21 @@
+import json
+
+from ..teaspoons_smoke_test_case import TeaspoonsSmokeTestCase
+
+
+class TeaspoonsVersionTests(TeaspoonsSmokeTestCase):
+    '''
+    Test the version endpoint for a 200 status code and that 'build' is populated in the response
+    '''
+    @staticmethod
+    def version_url() -> str:
+        return TeaspoonsSmokeTestCase.build_teaspoons_url("/version")
+
+    def test_status_code_is_200(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.version_url())
+        self.assertEqual(response.status_code, 200)
+
+    def test_version_value_specified(self):
+        response = TeaspoonsSmokeTestCase.call_teaspoons(self.version_url())
+        version = json.loads(response.text)
+        self.assertIsNotNone(version["build"], "build value must be non-empty")