Adopt mypy (#6821, #4558)

.github/workflows/ci.yml

@@ -59,6 +59,7 @@ jobs:
           make anvil_schema
           make check_clean
           make pep8
+          mypy
           AZUL_DEBUG=0 GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make test
           make check_clean
           coverage xml

.gitignore

@@ -1,6 +1,7 @@
 # Python
 *.pyc
 __pycache__/
+.mypy_cache/
 # Python coverage
 /.coverage
 /.coverage.*

.gitlab-ci.yml

@@ -90,6 +90,7 @@ test:
     - make anvil_schema
     - make check_clean
     - make pep8
+    - mypy
     - AZUL_DEBUG=0 make test
 
 deploy:

.mypy.ini

@@ -0,0 +1,22 @@
+[mypy]
+warn_unused_configs = True
+allow_redefinition = True
+explicit_package_bases = True
+modules =
+    azul.types,
+    azul.collections,
+    azul.args,
+    azul.attrs,
+    azul.auth,
+    azul.azulclient,
+    azul.bytes,
+    azul.caching,
+    azul.chalice
+packages =
+    azul.openapi
+
+[mypy-furl.*]
+follow_untyped_imports = True
+
+[mypy-requests.*]
+follow_untyped_imports = True

environment.py

@@ -678,7 +678,7 @@ def env() -> Mapping[str, Optional[str]]:
         'azul_gitlab_user': None,
 
         'PYTHONPATH': '{project_root}/src:{project_root}/test',
-        'MYPYPATH': '{project_root}/stubs',
+        'MYPYPATH': '{project_root}/src:{project_root}/stubs',
 
         # The path of a directory containing a wheel for each runtime
         # dependency. Settng this variable causes our fork of Chalice to skip

lambdas/indexer/app.py

@@ -111,7 +111,7 @@ def _authenticate(self) -> Optional[HMACAuthentication]:
 @app.route(
     '/{catalog}/{action}',
     methods=['POST'],
-    method_spec={
+    spec={
         'tags': ['Indexing'],
         'summary': 'Notify the indexer to perform an action on a bundle',
         'description': fd('''

lambdas/layer/app.py

@@ -14,6 +14,6 @@
                      spec={})
 
 
-@app.route('/', method_spec={})
+@app.route('/', spec={})
 def foo():
     pass

lambdas/service/app.py

@@ -474,7 +474,7 @@ def manifest_url(self,
     enabled=config.google_oauth2_client_id is not None,
     cache_control='no-store',
     interactive=False,
-    method_spec={
+    spec={
         'summary': 'Destination endpoint for Google OAuth 2.0 redirects',
         'tags': ['Auxiliary'],
         'responses': {
@@ -747,7 +747,7 @@ def fmt_error(err_description, params):
                 raise BRE(f'Invalid value for `{param_name}`')
 
 
-deprecated_method_spec = {
+deprecated_spec = {
     'summary': 'This endpoint will be removed in the future.',
     'tags': ['Deprecated'],
     'deprecated': True
@@ -758,7 +758,7 @@ def fmt_error(err_description, params):
     '/index/catalogs',
     methods=['GET'],
     cors=True,
-    method_spec={
+    spec={
         'summary': 'List all available catalogs.',
         'tags': ['Index'],
         'responses': {
@@ -789,10 +789,10 @@ def list_catalogs():
     return app.catalog_controller.list_catalogs()
 
 
-generic_object_spec = schema.object(additional_properties=True)
+generic_object_spec = schema.object(additionalProperties=True)
 array_of_object_spec = schema.array(generic_object_spec)
 hit_spec = schema.object(
-    additional_properties=True,
+    additionalProperties=True,
     protocols=array_of_object_spec,
     entryId=str,
     sources=array_of_object_spec,
@@ -815,7 +815,7 @@ def _filter_schema(field_type: FieldType) -> JSON:
     relations = field_type.supported_filter_relations
 
     def filter_schema(relation: str) -> JSON:
-        return schema.object_type(
+        return schema.object(
             properties={relation: schema.array(field_type.api_filter_schema(relation))},
             required=[relation],
             additionalProperties=False
@@ -831,7 +831,7 @@ def filter_schema(relation: str) -> JSON:
 
 filters_param_spec = params.query(
     'filters',
-    schema.optional(application_json(schema.object_type(
+    schema.optional(application_json(schema.object(
         default='{}',
         example={'cellCount': {'within': [[10000, 1000000000]]}},
         properties={
@@ -878,8 +878,8 @@ def filter_schema(relation: str) -> JSON:
 
 catalog_param_spec = params.query(
     'catalog',
-    schema.optional(schema.with_default(app.catalog,
-                                        type_=schema.enum(*config.catalogs))),
+    schema.optional(schema.default(app.catalog,
+                                   form=schema.enum(*config.catalogs))),
     description='The name of the catalog to query.')
 
 
@@ -894,7 +894,7 @@ def repository_search_params_spec():
         ),
         params.query(
             'size',
-            schema.optional(schema.with_default(10, type_=schema.in_range(min_page_size, None))),
+            schema.optional(schema.default(10, form=schema.range(min_page_size, None))),
             description=fd('''
                 The number of hits included per page. The maximum size allowed
                 depends on the catalog and entity type.
@@ -1074,7 +1074,7 @@ def repository_head_search_spec():
 @app.route(
     '/index/{entity_type}',
     methods=['GET'],
-    method_spec=repository_search_spec(post=False),
+    spec=repository_search_spec(post=False),
     cors=True
 )
 # FIXME: Properly document the POST version of /index
@@ -1083,19 +1083,19 @@ def repository_head_search_spec():
     '/index/{entity_type}',
     methods=['POST'],
     content_types=['application/json'],
-    method_spec=repository_search_spec(post=True),
+    spec=repository_search_spec(post=True),
     cors=True
 )
 @app.route(
     '/index/{entity_type}',
     methods=['HEAD'],
-    method_spec=repository_head_search_spec(),
+    spec=repository_head_search_spec(),
     cors=True
 )
 @app.route(
     '/index/{entity_type}/{entity_id}',
     methods=['GET'],
-    method_spec=repository_id_spec(),
+    spec=repository_id_spec(),
     cors=True
 )
 def repository_search(entity_type: str, entity_id: Optional[str] = None) -> JSON:
@@ -1128,7 +1128,7 @@ def _hoist_parameters(query_params, request):
     '/index/summary',
     methods=['GET'],
     cors=True,
-    method_spec={
+    spec={
         'summary': 'Statistics on the data present across all entities.',
         'responses': {
             '200': {
@@ -1152,7 +1152,7 @@ def _hoist_parameters(query_params, request):
                 '''),
                 **responses.json_content(
                     schema.object(
-                        additional_properties=True,
+                        additionalProperties=True,
                         organTypes=schema.array(str),
                         totalFileSize=float,
                         fileTypeSummaries=array_of_object_spec,
@@ -1173,7 +1173,7 @@ def _hoist_parameters(query_params, request):
 @app.route(
     '/index/summary',
     methods=['HEAD'],
-    method_spec={
+    spec={
         **repository_head_spec(for_summary=True),
         **repository_summary_spec
     }
@@ -1213,7 +1213,7 @@ def manifest_route(*, fetch: bool, initiate: bool):
                 '''))
             ]
         },
-        method_spec={
+        spec={
             'tags': ['Manifests'],
             'summary':
                 (
@@ -1302,7 +1302,7 @@ def manifest_route(*, fetch: bool, initiate: bool):
                                 format.value
                                 for format in app.metadata_plugin.manifest_formats
                             ],
-                            type_=str
+                            form=str
                         )
                     ),
                     description=f'''
@@ -1594,7 +1594,7 @@ def generate_manifest(event: AnyJSON, _context: LambdaContext):
     methods=['GET'],
     interactive=False,
     cors=True,
-    method_spec={
+    spec={
         **repository_files_spec,
         'summary': 'Redirect to a URL for downloading a given data file from the '
                    'underlying repository',
@@ -1661,7 +1661,7 @@ def repository_files(file_uuid: str) -> Response:
     '/fetch/repository/files/{file_uuid}',
     methods=['GET'],
     cors=True,
-    method_spec={
+    spec={
         **repository_files_spec,
         'summary': 'Request a URL for downloading a given data file',
         'responses': {
@@ -1747,7 +1747,7 @@ def validate_version(version: str) -> None:
     '/repository/sources',
     methods=['GET'],
     cors=True,
-    method_spec={
+    spec={
         'summary': 'List available data sources',
         'tags': ['Repository'],
         'parameters': [catalog_param_spec],
@@ -1799,7 +1799,7 @@ def hash_url(url):
     methods=['GET'],
     enabled=config.is_dss_enabled(),
     cors=True,
-    method_spec={
+    spec={
         'summary': 'Get file DRS object',
         'tags': ['DRS'],
         'description': fd('''
@@ -1847,7 +1847,7 @@ def get_data_object(file_uuid):
     methods=['GET'],
     enabled=config.is_dss_enabled(),
     cors=True,
-    method_spec={
+    spec={
         'summary': 'Get a file with an access ID',
         'description': fd('''
             This endpoint returns a URL that can be used to fetch the bytes of a
@@ -1898,7 +1898,7 @@ def get_data_object_access(file_uuid, access_id):
     methods=['GET'],
     enabled=config.is_dss_enabled(),
     cors=True,
-    method_spec=deprecated_method_spec
+    spec=deprecated_spec
 )
 def dos_get_data_object(file_uuid):
     """

requirements.all.txt

@@ -10,12 +10,12 @@ blinker==1.9.0
 boto3==1.35.84
 boto3-stubs==1.35.84
 botocore==1.35.84
-botocore-stubs==1.35.94
+botocore-stubs==1.36.2
 brotli==1.1.0
 cachetools==5.5.0
 certifi==2024.12.14
 cffi==1.17.1
-chalice==1.31.3+15
+chalice==1.31.3+16
 charset-normalizer==3.4.1
 chevron==0.14.0
 click==8.1.8
@@ -65,14 +65,15 @@ jmespath==1.0.1
 jq==1.8.0
 jsonschema==4.23.0
 jsonschema-path==0.3.3
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
 lazy-object-proxy==1.10.0
 locust==2.32.4
 markupsafe==3.0.2
 mccabe==0.7.0
 more-itertools==10.5.0
 moto==5.0.24
 msgpack==1.1.0
+mypy==1.14.1
 mypy-boto3-dynamodb==1.35.94
 mypy-boto3-ecr==1.35.93
 mypy-boto3-iam==1.35.93
@@ -81,12 +82,13 @@ mypy-boto3-lambda==1.35.93
 mypy-boto3-s3==1.35.93
 mypy-boto3-sqs==1.35.93
 mypy-boto3-stepfunctions==1.35.93
-openapi-schema-validator==0.6.2
+mypy-extensions==1.0.0
+openapi-schema-validator==0.6.3
 openapi-spec-validator==0.7.1
 openpyxl==3.1.5
 orderedmultidict==1.0.1
 packaging==24.2
-pathable==0.4.3
+pathable==0.4.4
 pip==24.3.1
 posix_ipc==1.1.1
 proto-plus==1.25.0
@@ -101,7 +103,7 @@ pyflakes==3.2.0
 pygithub==2.5.0
 pyjwt==2.10.1
 pynacl==1.5.0
-pyopenssl==24.3.0
+pyopenssl==25.0.0
 pyparsing==3.2.1
 python-dateutil==2.9.0.post0
 python-dxf==12.1.0
@@ -127,7 +129,8 @@ smmap==5.0.2
 strict-rfc3339==0.7
 tqdm==4.67.1
 types-awscrt==0.23.6
-types-s3transfer==0.10.4
+types-chevron==0.14.0
+types-s3transfer==0.11.1
 types-urllib3==1.26.20
 typing_extensions==4.12.2
 tzlocal==5.2
@@ -137,7 +140,7 @@ watchdog==6.0.0
 wcwidth==0.2.13
 werkzeug==3.1.3
 wheel==0.45.1
-wrapt==1.17.0
+wrapt==1.17.2
 www-authenticate==0.9.2
 xmltodict==0.14.2
 xmod==1.8.1

requirements.dev.trans.txt

@@ -1,6 +1,6 @@
 blessed==1.20.0
 blinker==1.9.0
-botocore-stubs==1.35.94
+botocore-stubs==1.36.2
 brotli==1.1.0
 click==8.1.8
 colorama==0.4.6
@@ -21,7 +21,7 @@ itsdangerous==2.2.0
 jinja2==3.1.5
 jsonschema==4.23.0
 jsonschema-path==0.3.3
-jsonschema-specifications==2023.12.1
+jsonschema-specifications==2024.10.1
 lazy-object-proxy==1.10.0
 mccabe==0.7.0
 mypy-boto3-dynamodb==1.35.94
@@ -32,8 +32,9 @@ mypy-boto3-lambda==1.35.93
 mypy-boto3-s3==1.35.93
 mypy-boto3-sqs==1.35.93
 mypy-boto3-stepfunctions==1.35.93
-openapi-schema-validator==0.6.2
-pathable==0.4.3
+mypy-extensions==1.0.0
+openapi-schema-validator==0.6.3
+pathable==0.4.4
 psutil==6.1.1
 py-partiql-parser==0.5.6
 pycodestyle==2.12.1
@@ -51,7 +52,7 @@ runs==1.2.2
 smmap==5.0.2
 tqdm==4.67.1
 types-awscrt==0.23.6
-types-s3transfer==0.10.4
+types-s3transfer==0.11.1
 uritemplate==4.1.1
 wcwidth==0.2.13
 www-authenticate==0.9.2