Dependabot Updates (DataBiosphere#3851)

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Michael R. Crusoe <[email protected]>
Co-authored-by: Lon Blauvelt <[email protected]>

requirements-aws.txt

@@ -2,4 +2,3 @@ boto>=2.48.0, <3
 boto3>=1.17, <2
 boto3-stubs[s3]>=1.17, <2
 futures>=3.1.1, <4
-pycryptodome==3.5.1

requirements-dev.txt

@@ -1,20 +1,23 @@
-mock==4.0.3
-pytest==6.2.1
-pytest-cov==2.11.1
-pytest-timeout==1.4.2
-stubserver==1.1
-setuptools==51.3.3
-sphinx==3.5.3
+mock>=4.0.3,<5
+pytest>=6.2.1,<7
+pytest-cov>=2.12.1,<4
+pytest-timeout>=1.4.2,<3
+stubserver>=1.1,<2
+setuptools>=51.3.3,<59
+sphinx>=3.5.3,<4
 cwltest==2.1.20210626101542
-mypy==0.812
-flake8==3.8.4
-flake8-bugbear==20.11.1
+mypy==0.910
+types-requests
+types-setuptools
+types-boto
+flake8==3.8.4,<5
+flake8-bugbear>=20.11.1,<21
 black
 isort
 pydocstyle
 autoflake
 isort
 diff_cover
 lxml
-docutils==0.16
+docutils>=0.16,<0.18
 pyupgrade

requirements-encryption.txt

@@ -1 +1 @@
-pynacl==1.3.0
+pynacl>=1.4.0,<2

requirements-google.txt

@@ -1,3 +1,2 @@
-gcs_oauth2_boto_plugin==1.14
-apache-libcloud==2.2.1
+apache-libcloud>=2.2.1,<3
 google-cloud-storage==1.6.0

requirements-htcondor.txt

@@ -1 +1 @@
-htcondor>=8.6.0
+htcondor>=8.6.0,<9

requirements-kubernetes.txt

@@ -1,3 +1,3 @@
-kubernetes>=12.0.1, <13
+kubernetes>=12.0.1, <=19
 idna>=2
 # Kubernetes's urllib3 can mange to use idna without really depending on it.

requirements-mesos.txt

@@ -1 +1 @@
-pymesos==0.3.15
+pymesos>=0.3.15,<0.4

setup.py

@@ -17,18 +17,26 @@
 from tempfile import NamedTemporaryFile
 from setuptools import find_packages, setup
 
+def get_requirements(extra=None):
+    """
+    Load the requirements for the given extra from the appropriate
+    requirements-extra.txt, or the main requirements.txt if no extra is
+    specified.
+    """
 
-cwltool_version = "3.1.20211004060744"
+    filename = f"requirements-{extra}.txt" if extra else "requirements.txt"
 
+    with open(filename) as fp:
+        # Parse out as one per line
+        return [l.strip() for l in fp.readlines() if l.strip()]
 
 def run_setup():
     """
     Calls setup(). This function exists so the setup() invocation preceded more internal
     functionality. The `version` module is imported dynamically by import_version() below.
     """
 
-    with open("requirements.txt") as fp:
-        install_requires = fp.read()
+    install_requires = get_requirements()
 
     extras_require = {}
     # htcondor is not supported by apple
@@ -46,11 +54,10 @@ def run_setup():
         "wdl",
     ]
     for extra in non_htcondor_extras:
-        with open(f"requirements-{extra}.txt") as fp:
-            extras_require[extra] = fp.read()
-            all_reqs += "\n" + extras_require[extra]
-    with open("requirements-htcondor.txt") as htcondor_fp:
-        extras_require['htcondor:sys_platform!="darwin"'] = htcondor_fp.read()
+        extras_require[extra] = get_requirements(extra)
+        all_reqs += "\n" + "\n".join(extras_require[extra])
+    # We exclude htcondor from "all" because it can't be on Mac
+    extras_require['htcondor:sys_platform!="darwin"'] = get_requirements("htcondor")
     extras_require["all"] = all_reqs
 
 
@@ -110,6 +117,11 @@ def run_setup():
 def import_version():
     """Return the module object for src/toil/version.py, generate from the template if required."""
     if not os.path.exists('src/toil/version.py'):
+        for req in get_requirements("cwl"):
+            # Determine cwltool version from requirements file
+            if req.startswith("cwltool=="):
+                cwltool_version = req[len("cwltool=="):]
+                break
         # Use the template to generate src/toil/version.py
         import version_template
         with NamedTemporaryFile(mode='w', dir='src/toil', prefix='version.py.', delete=False) as f:

src/toil/statsAndLogging.py

@@ -286,7 +286,7 @@ def suppress_exotic_logging(local_logger: str) -> None:
     top_level_loggers: List[str] = []
 
     # Due to https://stackoverflow.com/questions/61683713
-    for pkg_logger in list(logging.Logger.manager.loggerDict.keys()) + always_suppress: # type: ignore
+    for pkg_logger in list(logging.Logger.manager.loggerDict.keys()) + always_suppress:
         if pkg_logger != local_logger:
             # many sub-loggers may exist, like "boto.a", "boto.b", "boto.c"; we only want the top_level: "boto"
             top_level_logger = pkg_logger.split('.')[0] if '.' in pkg_logger else pkg_logger