update_job_status() now has option to ignore runtime errors

.github/workflows/qiita-ci.yml

@@ -7,11 +7,12 @@ on:
 jobs:
   # derived from https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml
   main:
+    # 7/16/24: confirm current ubuntu-latest is still 22.04.
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        python-version: ["2.7", "3.9"]
+        python-version: ["3.9"]
 
     services:
       postgres:
@@ -66,6 +67,7 @@ jobs:
           conda config --add channels conda-forge
           conda create -q --yes -n qiita python=3.9 libgfortran numpy nginx cython redis
           conda activate qiita
+          python -m pip install --upgrade pip
 
       - name: Qiita install
         shell: bash -l {0}
@@ -79,14 +81,14 @@ jobs:
         run: |
           conda create --yes -n qiita_client python=${{ matrix.python-version }} pip nose flake8 coverage
           conda activate qiita_client
-
-          pip --quiet install .
+          python -m pip install --upgrade pip
+          pip install .
 
       - name: Starting Main Services
         shell: bash -l {0}
         run: |
           conda activate qiita
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt
+          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
           sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP}
 
@@ -118,12 +120,17 @@ jobs:
           COVER_PACKAGE: ${{ matrix.cover_package }}
         run: |
           conda activate qiita_client
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/server.crt
+          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_server.crt
+          export QIITA_ROOT_CA=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
-
-          nosetests --with-doctest --with-coverage --cover-package=qiita_client
-
-      - uses: codecov/codecov-action@v1
+          export PYTHONWARNINGS="ignore:Certificate for localhost has no \`subjectAltName\`"
+
+          # before starting nosetests, add the root ca used to sign qiita's ci_server.crt file
+          # to the environment's certifi store. This will prevent CERTIFICATE_VERIFY_FAILED
+          # errors.
+          python rootca_insert.py $QIITA_ROOT_CA
+          nosetests --with-doctest --with-coverage -v --cover-package=qiita_client 
+      - uses: codecov/codecov-action@v3
         with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: codecov.yml

qiita_client/plugin.py

@@ -245,7 +245,12 @@ def __call__(self, server_url, job_id, output_dir):
 
         qclient = QiitaClient(server_url, config.get('oauth2', 'CLIENT_ID'),
                               config.get('oauth2', 'CLIENT_SECRET'),
-                              server_cert=config.get('oauth2', 'SERVER_CERT'))
+                              # for this group of tests, confirm optional
+                              # ca_cert parameter works as intended. Setting
+                              # this value will prevent underlying libraries
+                              # from validating the server's cert using
+                              # certifi's pem cache.
+                              ca_cert=environ['QIITA_ROOT_CA'])
 
         if job_id == 'register':
             self._register(qclient)

qiita_client/qiita_client.py

@@ -57,7 +57,7 @@ def __init__(self, output_name, artifact_type, files, archive=None):
 
     def __eq__(self, other):
         logger.debug('Entered ArtifactInfo.__eq__()')
-        if type(self) != type(other):
+        if type(self) != type(other):   # noqa
             return False
         if self.output_name != other.output_name or \
                 self.artifact_type != other.artifact_type or \
@@ -166,16 +166,16 @@ class QiitaClient(object):
         The client id to connect to the Qiita server
     client_secret : str
         The client secret id to connect to the Qiita server
-    server_cert : str, optional
-        The server certificate, in case that it is not verified
+    ca_cert : str, optional
+        CA cert used to sign and verify cert@server_url
 
 
     Methods
     -------
     get
     post
     """
-    def __init__(self, server_url, client_id, client_secret, server_cert=None):
+    def __init__(self, server_url, client_id, client_secret, ca_cert=None):
         self._server_url = server_url
         self._session = requests.Session()
 
@@ -186,16 +186,21 @@ def __init__(self, server_url, client_id, client_secret, server_cert=None):
         # if certificate verification should be performed or not, or a
         # string with the path to the certificate file that needs to be used
         # to verify the identity of the server.
-        # We are setting this attribute at __init__ time so we can avoid
-        # executing this if statement for each request issued.
-        if not server_cert:
+        # We are setting this attribute at __init__ time to avoid executing
+        # this if statement for each request issued.
+
+        # As self-signed server certs are no longer allowed in one or more of
+        # our dependencies, ca_cert (if provided) must now reference a file
+        # that can be used to verify the certificate used by the server
+        # referenced by server_url, rather than the server's own certificate.
+        if not ca_cert:
             # The server certificate is not provided, use standard certificate
             # verification methods
             self._verify = True
         else:
             # The server certificate is provided, use it to verify the identity
             # of the server
-            self._verify = server_cert
+            self._verify = ca_cert
 
         # Set up oauth2
         self._client_id = client_id
@@ -527,7 +532,7 @@ def get_job_info(self, job_id):
         logger.debug('Entered QiitaClient.get_job_info()')
         return self.get("/qiita_db/jobs/%s" % job_id)
 
-    def update_job_step(self, job_id, new_step):
+    def update_job_step(self, job_id, new_step, ignore_error=False):
         """Updates the current step of the job in the server
 
         Parameters
@@ -536,10 +541,16 @@ def update_job_step(self, job_id, new_step):
             The job id
         new_step : str
             The new step
+        ignore_error : bool
+            Problems communicating w/Qiita will not raise an Error.
         """
         logger.debug('Entered QiitaClient.update_job_step()')
         json_payload = dumps({'step': new_step})
-        self.post("/qiita_db/jobs/%s/step/" % job_id, data=json_payload)
+        try:
+            self.post("/qiita_db/jobs/%s/step/" % job_id, data=json_payload)
+        except BaseException as e:
+            if ignore_error is False:
+                raise e
 
     def complete_job(self, job_id, success, error_msg=None,
                      artifacts_info=None):

qiita_client/testing.py

@@ -24,11 +24,12 @@ def setUpClass(cls):
         cls.client_id = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4'
         cls.client_secret = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh'
                              'AmmCWZuabe0O5Mp28s1')
-        cls.server_cert = environ.get('QIITA_SERVER_CERT', None)
         qiita_port = int(environ.get('QIITA_PORT', 21174))
-        cls.qclient = QiitaClient(
-            "https://localhost:%d" % qiita_port, cls.client_id,
-            cls.client_secret, server_cert=cls.server_cert)
+
+        # do not rely on defining ca_cert for these tests. Instead append
+        # the appropriate CA cert to certifi's pem file.
+        cls.qclient = QiitaClient("https://localhost:%d" % qiita_port,
+                                  cls.client_id, cls.client_secret)
         logger.debug(
             'PluginTestCase.setUpClass() token %s' % cls.qclient._token)
         cls.qclient.post('/apitest/reload_plugins/')

qiita_client/tests/test_plugin.py

@@ -168,8 +168,7 @@ def html_generator_func(a, b, c, d):
                                  validate_func, html_generator_func, atypes)
 
         # Generate the config file for the new plugin
-        tester.generate_config('ls', 'echo',
-                               server_cert=self.server_cert)
+        tester.generate_config('ls', 'echo')
         # Ask Qiita to reload the plugins
         self.qclient.post('/apitest/reload_plugins/')
 
@@ -213,8 +212,7 @@ def func(qclient, job_id, job_params, working_dir):
                              {'out1': 'Demultiplexed'})
         tester.register_command(a_cmd)
 
-        tester.generate_config('ls', 'echo',
-                               server_cert=self.server_cert)
+        tester.generate_config('ls', 'echo')
         self.qclient.post('/apitest/reload_plugins/')
         tester("https://localhost:21174", 'register', 'ignored')
 

qiita_client/tests/test_qiita_client.py

@@ -7,7 +7,7 @@
 # -----------------------------------------------------------------------------
 
 from unittest import TestCase, main
-from os import environ, remove, close
+from os import remove, close
 from os.path import basename, exists
 from tempfile import mkstemp
 from json import dumps
@@ -19,6 +19,7 @@
 from qiita_client.exceptions import BadRequestError
 
 CLIENT_ID = '19ndkO3oMKsoChjVVWluF7QkxHRfYhTKSFbAVt8IhK7gZgDaO4'
+BAD_CLIENT_ID = 'NOT_A_CLIENT_ID'
 CLIENT_SECRET = ('J7FfQ7CQdOxuKhQAf1eoGgBAE81Ns8Gu3EKaWFm3IO2JKh'
                  'AmmCWZuabe0O5Mp28s1')
 
@@ -96,9 +97,13 @@ def test_format_payload_error(self):
 
 class QiitaClientTests(PluginTestCase):
     def setUp(self):
-        self.server_cert = environ.get('QIITA_SERVER_CERT', None)
-        self.tester = QiitaClient("https://localhost:21174", CLIENT_ID,
-                                  CLIENT_SECRET, server_cert=self.server_cert)
+        # self.server_cert = environ.get('QIITA_SERVER_CERT', None)
+        self.tester = QiitaClient("https://localhost:21174",
+                                  CLIENT_ID,
+                                  CLIENT_SECRET)
+        self.bad_tester = QiitaClient("https://localhost:21174",
+                                      BAD_CLIENT_ID,
+                                      CLIENT_SECRET)
         self.clean_up_files = []
 
         # making assertRaisesRegex compatible with Python 2.7 and 3.9
@@ -111,12 +116,11 @@ def tearDown(self):
                 remove(fp)
 
     def test_init(self):
-        obs = QiitaClient("https://localhost:21174", CLIENT_ID,
-                          CLIENT_SECRET, server_cert=self.server_cert)
+        obs = QiitaClient("https://localhost:21174", CLIENT_ID, CLIENT_SECRET)
         self.assertEqual(obs._server_url, "https://localhost:21174")
         self.assertEqual(obs._client_id, CLIENT_ID)
         self.assertEqual(obs._client_secret, CLIENT_SECRET)
-        self.assertEqual(obs._verify, self.server_cert)
+        self.assertEqual(obs._verify, True)
 
     def test_get(self):
         obs = self.tester.get("/qiita_db/artifacts/1/")
@@ -232,11 +236,36 @@ def test_update_job_step(self):
         obs = self.tester.update_job_step(job_id, new_step)
         self.assertIsNone(obs)
 
+    def test_update_job_step_ignore_failure(self):
+        job_id = "bcc7ebcd-39c1-43e4-af2d-822e3589f14d"
+        new_step = "some new step"
+
+        # confirm that update_job_step behaves as before when ignore_error
+        # parameter absent or set to False.
+
+        with self.assertRaises(BaseException):
+            self.bad_tester.update_job_step(job_id, new_step)
+
+        with self.assertRaises(BaseException):
+            self.bad_tester.update_job_step(job_id,
+                                            new_step,
+                                            ignore_error=False)
+
+        # confirm that when ignore_error is set to True, an Error is NOT
+        # raised.
+        try:
+            self.bad_tester.update_job_step(job_id,
+                                            new_step,
+                                            ignore_error=True)
+        except BaseException as e:
+            self.fail("update_job_step() raised an error: %s" % str(e))
+
     def test_complete_job(self):
         # Create a new job
         data = {
             'user': '[email protected]',
-            'command': dumps(['QIIME', '1.9.1', 'Pick closed-reference OTUs']),
+            'command': dumps(['QIIMEq2', '1.9.1',
+                              'Pick closed-reference OTUs']),
             'status': 'running',
             'parameters': dumps({"reference": 1,
                                  "sortmerna_e_value": 1,

rootca_insert.py

@@ -0,0 +1,18 @@
+import certifi
+from sys import argv
+
+
+def main(ca_fp):
+    with open(ca_fp, 'rb') as f:
+        my_ca = f.read()
+
+    ca_file = certifi.where()
+
+    with open(ca_file, 'ab') as f:
+        f.write(my_ca)
+
+    print("%s updated" % ca_file)
+
+
+if __name__ == '__main__':
+    main(argv[1])