diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6f33fdb69..5f0dc86f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: make anvil_schema make check_clean make pep8 + mypy AZUL_DEBUG=0 GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make test make check_clean coverage xml diff --git a/.gitignore b/.gitignore index 40a8a7d646..6ef15399d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Python *.pyc __pycache__/ +.mypy_cache/ # Python coverage /.coverage /.coverage.* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdcf66b823..fa351a1666 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -90,6 +90,7 @@ test: - make anvil_schema - make check_clean - make pep8 + - mypy - AZUL_DEBUG=0 make test deploy: diff --git a/.mypy.ini b/.mypy.ini new file mode 100644 index 0000000000..9fa196d4e8 --- /dev/null +++ b/.mypy.ini @@ -0,0 +1,22 @@ +[mypy] +warn_unused_configs = True +allow_redefinition = True +explicit_package_bases = True +modules = + azul.types, + azul.collections, + azul.args, + azul.attrs, + azul.auth, + azul.azulclient, + azul.bytes, + azul.caching, + azul.chalice +packages = + azul.openapi + +[mypy-furl.*] +follow_untyped_imports = True + +[mypy-requests.*] +follow_untyped_imports = True \ No newline at end of file diff --git a/bin/wheels/runtime/pyOpenSSL-24.3.0-py3-none-any.whl b/bin/wheels/runtime/pyOpenSSL-24.3.0-py3-none-any.whl deleted file mode 100644 index 6519fbcf5d..0000000000 Binary files a/bin/wheels/runtime/pyOpenSSL-24.3.0-py3-none-any.whl and /dev/null differ diff --git a/bin/wheels/runtime/pyOpenSSL-25.0.0-py3-none-any.whl b/bin/wheels/runtime/pyOpenSSL-25.0.0-py3-none-any.whl new file mode 100644 index 0000000000..cd514343ba Binary files /dev/null and b/bin/wheels/runtime/pyOpenSSL-25.0.0-py3-none-any.whl differ diff --git a/bin/wheels/runtime/wrapt-1.17.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/wrapt-1.17.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl deleted file mode 100644 index 306d10a965..0000000000 Binary files a/bin/wheels/runtime/wrapt-1.17.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl and /dev/null differ diff --git a/bin/wheels/runtime/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl b/bin/wheels/runtime/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl new file mode 100644 index 0000000000..e78a278978 Binary files /dev/null and b/bin/wheels/runtime/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl differ diff --git a/environment.py b/environment.py index e35ae55d4c..b09fb687f9 100644 --- a/environment.py +++ b/environment.py @@ -678,7 +678,7 @@ def env() -> Mapping[str, Optional[str]]: 'azul_gitlab_user': None, 'PYTHONPATH': '{project_root}/src:{project_root}/test', - 'MYPYPATH': '{project_root}/stubs', + 'MYPYPATH': '{project_root}/src:{project_root}/stubs', # The path of a directory containing a wheel for each runtime # dependency. Settng this variable causes our fork of Chalice to skip diff --git a/lambdas/indexer/app.py b/lambdas/indexer/app.py index 2d33bb59a7..612a28fd03 100644 --- a/lambdas/indexer/app.py +++ b/lambdas/indexer/app.py @@ -111,7 +111,7 @@ def _authenticate(self) -> Optional[HMACAuthentication]: @app.route( '/{catalog}/{action}', methods=['POST'], - method_spec={ + spec={ 'tags': ['Indexing'], 'summary': 'Notify the indexer to perform an action on a bundle', 'description': fd(''' diff --git a/lambdas/layer/app.py b/lambdas/layer/app.py index f0370973cb..7feec21189 100644 --- a/lambdas/layer/app.py +++ b/lambdas/layer/app.py @@ -14,6 +14,6 @@ spec={}) -@app.route('/', method_spec={}) +@app.route('/', spec={}) def foo(): pass diff --git a/lambdas/service/app.py b/lambdas/service/app.py index b70f56b373..19263415c7 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -474,7 +474,7 @@ def manifest_url(self, enabled=config.google_oauth2_client_id is not None, cache_control='no-store', interactive=False, - method_spec={ + spec={ 'summary': 'Destination endpoint for Google OAuth 2.0 redirects', 'tags': ['Auxiliary'], 'responses': { @@ -747,7 +747,7 @@ def fmt_error(err_description, params): raise BRE(f'Invalid value for `{param_name}`') -deprecated_method_spec = { +deprecated_spec = { 'summary': 'This endpoint will be removed in the future.', 'tags': ['Deprecated'], 'deprecated': True @@ -758,7 +758,7 @@ def fmt_error(err_description, params): '/index/catalogs', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'List all available catalogs.', 'tags': ['Index'], 'responses': { @@ -789,10 +789,10 @@ def list_catalogs(): return app.catalog_controller.list_catalogs() -generic_object_spec = schema.object(additional_properties=True) +generic_object_spec = schema.object(additionalProperties=True) array_of_object_spec = schema.array(generic_object_spec) hit_spec = schema.object( - additional_properties=True, + additionalProperties=True, protocols=array_of_object_spec, entryId=str, sources=array_of_object_spec, @@ -815,7 +815,7 @@ def _filter_schema(field_type: FieldType) -> JSON: relations = field_type.supported_filter_relations def filter_schema(relation: str) -> JSON: - return schema.object_type( + return schema.object( properties={relation: schema.array(field_type.api_filter_schema(relation))}, required=[relation], additionalProperties=False @@ -831,7 +831,7 @@ def filter_schema(relation: str) -> JSON: filters_param_spec = params.query( 'filters', - schema.optional(application_json(schema.object_type( + schema.optional(application_json(schema.object( default='{}', example={'cellCount': {'within': [[10000, 1000000000]]}}, properties={ @@ -878,8 +878,8 @@ def filter_schema(relation: str) -> JSON: catalog_param_spec = params.query( 'catalog', - schema.optional(schema.with_default(app.catalog, - type_=schema.enum(*config.catalogs))), + schema.optional(schema.default(app.catalog, + form=schema.enum(*config.catalogs))), description='The name of the catalog to query.') @@ -894,7 +894,7 @@ def repository_search_params_spec(): ), params.query( 'size', - schema.optional(schema.with_default(10, type_=schema.in_range(min_page_size, None))), + schema.optional(schema.default(10, form=schema.range(min_page_size, None))), description=fd(''' The number of hits included per page. The maximum size allowed depends on the catalog and entity type. @@ -1074,7 +1074,7 @@ def repository_head_search_spec(): @app.route( '/index/{entity_type}', methods=['GET'], - method_spec=repository_search_spec(post=False), + spec=repository_search_spec(post=False), cors=True ) # FIXME: Properly document the POST version of /index @@ -1083,19 +1083,19 @@ def repository_head_search_spec(): '/index/{entity_type}', methods=['POST'], content_types=['application/json'], - method_spec=repository_search_spec(post=True), + spec=repository_search_spec(post=True), cors=True ) @app.route( '/index/{entity_type}', methods=['HEAD'], - method_spec=repository_head_search_spec(), + spec=repository_head_search_spec(), cors=True ) @app.route( '/index/{entity_type}/{entity_id}', methods=['GET'], - method_spec=repository_id_spec(), + spec=repository_id_spec(), cors=True ) def repository_search(entity_type: str, entity_id: Optional[str] = None) -> JSON: @@ -1128,7 +1128,7 @@ def _hoist_parameters(query_params, request): '/index/summary', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Statistics on the data present across all entities.', 'responses': { '200': { @@ -1152,7 +1152,7 @@ def _hoist_parameters(query_params, request): '''), **responses.json_content( schema.object( - additional_properties=True, + additionalProperties=True, organTypes=schema.array(str), totalFileSize=float, fileTypeSummaries=array_of_object_spec, @@ -1173,7 +1173,7 @@ def _hoist_parameters(query_params, request): @app.route( '/index/summary', methods=['HEAD'], - method_spec={ + spec={ **repository_head_spec(for_summary=True), **repository_summary_spec } @@ -1213,7 +1213,7 @@ def manifest_route(*, fetch: bool, initiate: bool): ''')) ] }, - method_spec={ + spec={ 'tags': ['Manifests'], 'summary': ( @@ -1302,7 +1302,7 @@ def manifest_route(*, fetch: bool, initiate: bool): format.value for format in app.metadata_plugin.manifest_formats ], - type_=str + form=str ) ), description=f''' @@ -1594,7 +1594,7 @@ def generate_manifest(event: AnyJSON, _context: LambdaContext): methods=['GET'], interactive=False, cors=True, - method_spec={ + spec={ **repository_files_spec, 'summary': 'Redirect to a URL for downloading a given data file from the ' 'underlying repository', @@ -1661,7 +1661,7 @@ def repository_files(file_uuid: str) -> Response: '/fetch/repository/files/{file_uuid}', methods=['GET'], cors=True, - method_spec={ + spec={ **repository_files_spec, 'summary': 'Request a URL for downloading a given data file', 'responses': { @@ -1747,7 +1747,7 @@ def validate_version(version: str) -> None: '/repository/sources', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'List available data sources', 'tags': ['Repository'], 'parameters': [catalog_param_spec], @@ -1799,7 +1799,7 @@ def hash_url(url): methods=['GET'], enabled=config.is_dss_enabled(), cors=True, - method_spec={ + spec={ 'summary': 'Get file DRS object', 'tags': ['DRS'], 'description': fd(''' @@ -1847,7 +1847,7 @@ def get_data_object(file_uuid): methods=['GET'], enabled=config.is_dss_enabled(), cors=True, - method_spec={ + spec={ 'summary': 'Get a file with an access ID', 'description': fd(''' This endpoint returns a URL that can be used to fetch the bytes of a @@ -1898,7 +1898,7 @@ def get_data_object_access(file_uuid, access_id): methods=['GET'], enabled=config.is_dss_enabled(), cors=True, - method_spec=deprecated_method_spec + spec=deprecated_spec ) def dos_get_data_object(file_uuid): """ diff --git a/requirements.all.txt b/requirements.all.txt index be39536901..076124408e 100644 --- a/requirements.all.txt +++ b/requirements.all.txt @@ -10,12 +10,12 @@ blinker==1.9.0 boto3==1.35.84 boto3-stubs==1.35.84 botocore==1.35.84 -botocore-stubs==1.35.94 +botocore-stubs==1.36.2 brotli==1.1.0 cachetools==5.5.0 certifi==2024.12.14 cffi==1.17.1 -chalice==1.31.3+15 +chalice==1.31.3+16 charset-normalizer==3.4.1 chevron==0.14.0 click==8.1.8 @@ -65,7 +65,7 @@ jmespath==1.0.1 jq==1.8.0 jsonschema==4.23.0 jsonschema-path==0.3.3 -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 lazy-object-proxy==1.10.0 locust==2.32.4 markupsafe==3.0.2 @@ -73,6 +73,7 @@ mccabe==0.7.0 more-itertools==10.5.0 moto==5.0.24 msgpack==1.1.0 +mypy==1.14.1 mypy-boto3-dynamodb==1.35.94 mypy-boto3-ecr==1.35.93 mypy-boto3-iam==1.35.93 @@ -81,12 +82,13 @@ mypy-boto3-lambda==1.35.93 mypy-boto3-s3==1.35.93 mypy-boto3-sqs==1.35.93 mypy-boto3-stepfunctions==1.35.93 -openapi-schema-validator==0.6.2 +mypy-extensions==1.0.0 +openapi-schema-validator==0.6.3 openapi-spec-validator==0.7.1 openpyxl==3.1.5 orderedmultidict==1.0.1 packaging==24.2 -pathable==0.4.3 +pathable==0.4.4 pip==24.3.1 posix_ipc==1.1.1 proto-plus==1.25.0 @@ -101,7 +103,7 @@ pyflakes==3.2.0 pygithub==2.5.0 pyjwt==2.10.1 pynacl==1.5.0 -pyopenssl==24.3.0 +pyopenssl==25.0.0 pyparsing==3.2.1 python-dateutil==2.9.0.post0 python-dxf==12.1.0 @@ -127,7 +129,8 @@ smmap==5.0.2 strict-rfc3339==0.7 tqdm==4.67.1 types-awscrt==0.23.6 -types-s3transfer==0.10.4 +types-chevron==0.14.0 +types-s3transfer==0.11.1 types-urllib3==1.26.20 typing_extensions==4.12.2 tzlocal==5.2 @@ -137,7 +140,7 @@ watchdog==6.0.0 wcwidth==0.2.13 werkzeug==3.1.3 wheel==0.45.1 -wrapt==1.17.0 +wrapt==1.17.2 www-authenticate==0.9.2 xmltodict==0.14.2 xmod==1.8.1 diff --git a/requirements.dev.trans.txt b/requirements.dev.trans.txt index 6f4912ab60..0f3e93a276 100644 --- a/requirements.dev.trans.txt +++ b/requirements.dev.trans.txt @@ -1,6 +1,6 @@ blessed==1.20.0 blinker==1.9.0 -botocore-stubs==1.35.94 +botocore-stubs==1.36.2 brotli==1.1.0 click==8.1.8 colorama==0.4.6 @@ -21,7 +21,7 @@ itsdangerous==2.2.0 jinja2==3.1.5 jsonschema==4.23.0 jsonschema-path==0.3.3 -jsonschema-specifications==2023.12.1 +jsonschema-specifications==2024.10.1 lazy-object-proxy==1.10.0 mccabe==0.7.0 mypy-boto3-dynamodb==1.35.94 @@ -32,8 +32,9 @@ mypy-boto3-lambda==1.35.93 mypy-boto3-s3==1.35.93 mypy-boto3-sqs==1.35.93 mypy-boto3-stepfunctions==1.35.93 -openapi-schema-validator==0.6.2 -pathable==0.4.3 +mypy-extensions==1.0.0 +openapi-schema-validator==0.6.3 +pathable==0.4.4 psutil==6.1.1 py-partiql-parser==0.5.6 pycodestyle==2.12.1 @@ -51,7 +52,7 @@ runs==1.2.2 smmap==5.0.2 tqdm==4.67.1 types-awscrt==0.23.6 -types-s3transfer==0.10.4 +types-s3transfer==0.11.1 uritemplate==4.1.1 wcwidth==0.2.13 www-authenticate==0.9.2 diff --git a/requirements.dev.txt b/requirements.dev.txt index 6dabf2df37..9e762682c1 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -5,7 +5,7 @@ coverage==7.6.9 docker==7.1.0 flake8==7.1.1 gevent==24.11.1 -git+https://github.com/DataBiosphere/azul-chalice@1.31.3+15#egg=chalice +git+https://github.com/DataBiosphere/azul-chalice@1.31.3+16#egg=chalice git+https://github.com/hannes-ucsc/requirements-parser@v0.2.0+1#egg=requirements-parser gitpython==3.1.43 google-api-python-client==2.156.0 @@ -14,6 +14,7 @@ google-cloud-storage==2.19.0 jq==1.8.0 locust==2.32.4 moto[s3,sqs,sns,dynamodb,iam]==5.0.24 # match the extras with the backends listed in AzulUnitTestCase._reset_moto +mypy==1.14.1 openapi-spec-validator==0.7.1 openpyxl==3.1.5 posix_ipc==1.1.1 @@ -23,6 +24,7 @@ python-gitlab==5.2.0 pyyaml==6.0.2 responses==0.25.3 strict-rfc3339==0.7 +types-chevron==0.14.0 # match with chevron in requirements.txt types-urllib3==1.26.20 # match with urllib in requirements.txt watchdog==6.0.0 -r requirements.dev.trans.txt diff --git a/requirements.trans.txt b/requirements.trans.txt index fbde89dc99..8fb841e42d 100644 --- a/requirements.trans.txt +++ b/requirements.trans.txt @@ -21,7 +21,7 @@ protobuf==4.25.5 pyasn1==0.6.1 pyasn1_modules==0.4.1 pycparser==2.22 -pyopenssl==24.3.0 +pyopenssl==25.0.0 python-dateutil==2.9.0.post0 pytz==2024.2 s3transfer==0.10.4 @@ -29,4 +29,4 @@ setuptools-scm==5.0.2 six==1.17.0 typing_extensions==4.12.2 tzlocal==5.2 -wrapt==1.17.0 +wrapt==1.17.2 diff --git a/requirements.txt b/requirements.txt index 3ee15a4c95..1f9bc590ef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ aws-requests-auth==0.4.3 bdbag==1.7.3 boto3==1.35.84 # Match version of the `boto3-stubs` dev dependency botocore==1.35.84 -chevron==0.14.0 +chevron==0.14.0 # Match with types-chevron in requirements.dev.txt deprecated==1.2.15 elasticsearch==7.17.12 # 7.x to match server-side elasticsearch-dsl==7.4.1 # 7.x to match server-side diff --git a/src/azul/__init__.py b/src/azul/__init__.py index f33e5eacbc..641db20f87 100644 --- a/src/azul/__init__.py +++ b/src/azul/__init__.py @@ -23,14 +23,14 @@ from typing import ( BinaryIO, ClassVar, + Literal, NotRequired, - Optional, Self, TYPE_CHECKING, TextIO, TypeVar, TypedDict, - Union, + overload, ) import attr @@ -141,7 +141,7 @@ def _validate_debug(self, debug): _es_endpoint_env_name = 'AZUL_ES_ENDPOINT' @property - def es_endpoint(self) -> Optional[Netloc]: + def es_endpoint(self) -> Netloc | None: try: es_endpoint = self.environ[self._es_endpoint_env_name] except KeyError: @@ -152,8 +152,8 @@ def es_endpoint(self) -> Optional[Netloc]: def es_endpoint_env(self, *, - es_endpoint: Union[Netloc, str], - es_instance_count: Union[int, str] + es_endpoint: Netloc | str, + es_instance_count: int | str ) -> Mapping[str, str]: if isinstance(es_endpoint, tuple): host, port = es_endpoint @@ -217,7 +217,7 @@ def qualified_bucket_name(self, def alb_access_log_path_prefix(self, *component: str, - deployment: Optional[str] = current, + deployment: str | None = current, ) -> str: """ :param deployment: Which deployment name to use in the path. Omit this @@ -230,7 +230,7 @@ def alb_access_log_path_prefix(self, def s3_access_log_path_prefix(self, *component: str, - deployment: Optional[str] = current, + deployment: str | None = current, ) -> str: """ :param deployment: Which deployment name to use in the path. Omit this @@ -243,7 +243,7 @@ def s3_access_log_path_prefix(self, def _log_path_prefix(self, prefix: list[str], - deployment: Optional[str], + deployment: str | None, *component: str, ): if deployment is self.current: @@ -291,7 +291,7 @@ def data_browser_domain(self): return domain @property - def dss_endpoint(self) -> Optional[str]: + def dss_endpoint(self) -> str | None: if self.dss_source is None: return None else: @@ -301,7 +301,7 @@ def dss_endpoint(self) -> Optional[str]: return SimpleSourceSpec.parse(self.dss_source).name @property - def dss_source(self) -> Optional[str]: + def dss_source(self) -> str | None: return self.environ.get('AZUL_DSS_SOURCE') def sources(self, catalog: CatalogName) -> Set[str]: @@ -331,7 +331,7 @@ def sam_service_url(self) -> mutable_furl: return furl(self.environ['AZUL_SAM_SERVICE_URL']) @property - def duos_service_url(self) -> Optional[mutable_furl]: + def duos_service_url(self) -> mutable_furl | None: url = self.environ.get('AZUL_DUOS_SERVICE_URL') return None if url is None else furl(url) @@ -369,8 +369,8 @@ def dss_direct_access(self) -> bool: def dss_direct_access_role(self, lambda_name: str, - stage: Optional[str] = None - ) -> Optional[str]: + stage: str | None = None + ) -> str | None: key = 'AZUL_DSS_DIRECT_ACCESS_ROLE' try: role_arn = self.environ[key] @@ -677,13 +677,13 @@ def indexer_name(self) -> str: def service_name(self) -> str: return self.service_function_name() - def indexer_function_name(self, handler_name: Optional[str] = None): + def indexer_function_name(self, handler_name: str | None = None): return self._function_name('indexer', handler_name) - def service_function_name(self, handler_name: Optional[str] = None): + def service_function_name(self, handler_name: str | None = None): return self._function_name('service', handler_name) - def _function_name(self, lambda_name: str, handler_name: Optional[str]): + def _function_name(self, lambda_name: str, handler_name: str | None): if handler_name is None: return self.qualified_resource_name(lambda_name) else: @@ -883,7 +883,7 @@ def it_catalog(self) -> CatalogName: return name @classmethod - def from_json(cls, name: str, spec: JSON) -> 'Config.Catalog': + def from_json(cls, name: str, spec: JSON) -> Self: plugins = { plugin_type: cls.Plugin(**plugin_spec) for plugin_type, plugin_spec in spec['plugins'].items() @@ -925,29 +925,29 @@ def default_catalog(self) -> CatalogName: return first(self.catalogs) @property - def current_catalog(self) -> Optional[str]: + def current_catalog(self) -> str | None: return self.environ.get('azul_current_catalog') - def it_catalog_for(self, catalog: CatalogName) -> Optional[CatalogName]: + def it_catalog_for(self, catalog: CatalogName) -> CatalogName | None: it_catalog = self.catalogs[catalog].it_catalog assert it_catalog in self.integration_test_catalogs, it_catalog return it_catalog - def is_dss_enabled(self, catalog: Optional[str] = None) -> bool: + def is_dss_enabled(self, catalog: str | None = None) -> bool: return self._is_plugin_enabled('dss', catalog) - def is_tdr_enabled(self, catalog: Optional[str] = None) -> bool: + def is_tdr_enabled(self, catalog: str | None = None) -> bool: return self._is_plugin_enabled('tdr', catalog) - def is_hca_enabled(self, catalog: Optional[str] = None) -> bool: + def is_hca_enabled(self, catalog: str | None = None) -> bool: return self._is_plugin_enabled('hca', catalog) - def is_anvil_enabled(self, catalog: Optional[str] = None) -> bool: + def is_anvil_enabled(self, catalog: str | None = None) -> bool: return self._is_plugin_enabled('anvil', catalog) def _is_plugin_enabled(self, plugin_prefix: str, - catalog: Optional[str] + catalog: str | None ) -> bool: def predicate(catalog): return any( @@ -1078,7 +1078,7 @@ def deployment(self) -> Deployment: return self.Deployment(self.deployment_stage) @property - def _shared_deployments(self) -> Mapping[Optional[str], Sequence[Deployment]]: + def _shared_deployments(self) -> Mapping[str | None, Sequence[Deployment]]: """ Maps a branch name to a sequence of names of shared deployments the branch can be deployed to. The key of None signifies any other branch @@ -1299,7 +1299,7 @@ def secrets_manager_secret_name(self, *args): def enable_gcp(self): return self.google_project() is not None - def google_project(self) -> Optional[str]: + def google_project(self) -> str | None: return self.environ.get('GOOGLE_PROJECT') class ServiceAccount(Enum): @@ -1459,7 +1459,7 @@ def github_access_token(self) -> str: return self.environ['azul_github_access_token'] @property - def gitlab_access_token(self) -> Optional[str]: + def gitlab_access_token(self) -> str | None: return self.environ.get('azul_gitlab_access_token') @property @@ -1488,7 +1488,7 @@ def current_sources(self) -> list[str]: minimum_compression_size = 0 @property - def google_oauth2_client_id(self) -> Optional[str]: + def google_oauth2_client_id(self) -> str | None: return self.environ.get('AZUL_GOOGLE_OAUTH2_CLIENT_ID') @property @@ -1522,7 +1522,7 @@ class SlackIntegration: channel_id: str @property - def slack_integration(self) -> Optional[SlackIntegration]: + def slack_integration(self) -> SlackIntegration | None: # FIXME: Eliminate local import # https://github.com/DataBiosphere/azul/issues/3133 @@ -1682,9 +1682,24 @@ def reject(condition: bool, *args, exception: type = RequirementError): raise exception(*args) +@overload def open_resource(*path: str, - package_root: Optional[str] = None, - binary=False) -> Union[TextIO, BinaryIO]: + package_root: str | None = None, + binary: Literal[False] = False + ) -> TextIO: ... + + +@overload +def open_resource(*path: str, + package_root: str | None = None, + binary: Literal[True] = False + ) -> BinaryIO: ... + + +def open_resource(*path: str, + package_root: str | None = None, + binary: bool = False + ) -> TextIO | BinaryIO: """ Return a file object for the resources at the given path. A resource is a source file that can be loaded at runtime. Resources typically aren't @@ -1744,7 +1759,7 @@ def str_to_bool(string: str): E = TypeVar('E') -def iif(condition: bool, then: T, otherwise: E = absent) -> Union[T, E]: +def iif(condition: bool, then: T, otherwise: E = absent) -> T | E: """ An alternative to ``if`` expressions, that, in certain situations, might be more convenient or readable, such as when the ``else`` branch diff --git a/src/azul/attrs.py b/src/azul/attrs.py index 535a7289d5..06589673d0 100644 --- a/src/azul/attrs.py +++ b/src/azul/attrs.py @@ -52,8 +52,8 @@ def as_annotated(): ... # doctest: +NORMALIZE_WHITESPACE Traceback (most recent call last): ... - TypeError: ('y', set(), (, - , , , + TypeError: ('y', set(), (, + , , , , , )) Note that you cannot share one return value of this function between more @@ -111,13 +111,13 @@ def __call__(self, _instance, field, value): def _reify(self, field): # reify() isn't exactly cheap so we'll cache its result if self._cache is None: - reified_type = reify(field.type) - self._cache = field, reified_type + reified_types = reify(field.type) + self._cache = field, reified_types else: - cached_field, reified_type = self._cache + cached_field, reified_types = self._cache require(cached_field == field, 'Validator cannot be shared among fields', cached_field, field) - return reified_type + return reified_types def __repr__(self): return 'as_annotated()' diff --git a/src/azul/auth.py b/src/azul/auth.py index b843317764..a021397d20 100644 --- a/src/azul/auth.py +++ b/src/azul/auth.py @@ -7,7 +7,6 @@ ) from typing import ( ClassVar, - Type, ) import attr @@ -68,9 +67,10 @@ def to_json(self) -> JSON: def from_json(cls, json: JSON) -> 'Authentication': json = copy_json(json) cls_name = json.pop(cls._cls_field) + assert isinstance(cls_name, str) return cls._cls_for_name[cls_name](**json) - _cls_for_name: ClassVar[dict[str, Type['Authentication']]] = {} + _cls_for_name: ClassVar[dict[str, type['Authentication']]] = {} def __init_subclass__(cls) -> None: super().__init_subclass__() diff --git a/src/azul/azulclient.py b/src/azul/azulclient.py index 5917dfc533..377b18b6d8 100644 --- a/src/azul/azulclient.py +++ b/src/azul/azulclient.py @@ -94,7 +94,7 @@ def notification(self, bundle_fqid: SourcedBundleFQID) -> JSON: # only variant that would ever occur in the wild. return { 'transaction_id': str(uuid.uuid4()), - 'bundle_fqid': bundle_fqid.to_json() + 'bundle_fqid': cast(JSON, bundle_fqid.to_json()) } def bundle_message(self, @@ -115,7 +115,7 @@ def reindex_message(self, return { 'action': 'reindex', 'catalog': catalog, - 'source': source.to_json(), + 'source': cast(JSON, source.to_json()), 'prefix': prefix } @@ -133,7 +133,7 @@ def index(self, notifications: Iterable[JSON], delete: bool = False ): - errors = defaultdict(int) + errors = defaultdict[int, int](int) missing = [] indexed = 0 total = 0 @@ -247,16 +247,16 @@ def remote_reindex(self, sources: Set[str]): plugin = self.repository_plugin(catalog) - for source in sources: - source = plugin.resolve_source(source) - source = plugin.partition_source(catalog, source) + for source_spec in sources: + source_ref = plugin.resolve_source(source_spec) + source_ref = plugin.partition_source(catalog, source_ref) def message(partition_prefix: str) -> JSON: - log.info('Remotely reindexing prefix %r of source %r into catalog %r', - partition_prefix, str(source.spec), catalog) - return self.reindex_message(catalog, source, partition_prefix) + log.info('Remotely reindexing prefix %r of source_ref %r into catalog %r', + partition_prefix, str(source_ref.spec), catalog) + return self.reindex_message(catalog, source_ref, partition_prefix) - messages = map(message, source.spec.prefix.partition_prefixes()) + messages = map(message, source_ref.spec.prefix.partition_prefixes()) for batch in chunked(messages, 10): entries = [ dict(Id=str(i), MessageBody=json.dumps(message)) @@ -265,8 +265,8 @@ def message(partition_prefix: str) -> JSON: self.notifications_queue.send_messages(Entries=entries) def remote_reindex_partition(self, message: JSON) -> None: - catalog = message['catalog'] - prefix = message['prefix'] + catalog, prefix = message['catalog'], message['prefix'] + assert isinstance(catalog, str) and isinstance(prefix, str) # FIXME: Adopt `trycast` for casting JSON to TypeDict # https://github.com/DataBiosphere/azul/issues/5171 source = cast(SourceJSON, message['source']) @@ -386,11 +386,11 @@ def group_key(fqid: SourcedBundleFQID): fqid.uuid.lower() ) - bundle_fqids = groupby(bundle_fqids, key=group_key) + groups = groupby(bundle_fqids, key=group_key) # Take the first item in each group. Because the oder is reversed, this # is the latest version - bundle_fqids = [next(group) for _, group in bundle_fqids] + bundle_fqids = [next(group) for _, group in groups] return bundle_fqids @cached_property diff --git a/src/azul/chalice.py b/src/azul/chalice.py index 433fc5ff79..781686d0a4 100644 --- a/src/azul/chalice.py +++ b/src/azul/chalice.py @@ -17,6 +17,7 @@ import pathlib from typing import ( Any, + Callable, Iterator, Literal, Self, @@ -50,6 +51,7 @@ config, mutable_furl, open_resource, + reject, require, ) from azul.auth import ( @@ -80,6 +82,9 @@ from azul.types import ( JSON, LambdaContext, + json_dict, + json_list, + json_str, ) log = logging.getLogger(__name__) @@ -124,10 +129,8 @@ def aws_name(self) -> str: class AzulChaliceApp(Chalice): - # FIXME: Remove these two class attributes once upstream issue is fixed - # https://github.com/DataBiosphere/azul/issues/4558 - lambda_context = None - current_request = None + lambda_context: LambdaContext | None + current_request: AzulRequest | None def __init__(self, app_name: str, @@ -136,11 +139,11 @@ def __init__(self, unit_test: bool = False, spec: JSON): self._patch_event_source_handler() - assert app_module_path.endswith('/app.py'), app_module_path + require(app_module_path.endswith('/app.py'), app_module_path) self.app_module_path = app_module_path self.unit_test = unit_test self.non_interactive_routes: set[tuple[str, str]] = set() - assert 'paths' not in spec, 'The top-level spec must not define paths' + reject('paths' in spec, 'The top-level spec must not define paths') self._specs = copy_json(spec) self._specs['paths'] = {} # The `debug` arg controls whether tracebacks appear in error responses @@ -255,17 +258,17 @@ def _http_cache_for(self, seconds: int): HttpMethod = Literal['GET', 'POST', 'PUT', 'PATCH', 'HEAD', 'OPTIONS', 'DELETE'] - # noinspection PyMethodOverriding - def route(self, - path: str, - *, - methods: Sequence[HttpMethod] = ('GET',), - enabled: bool = True, - interactive: bool = True, - cache_control: str = 'no-store', - path_spec: JSON | None = None, - method_spec: JSON, - **kwargs): + def route[C: Callable](self, + path: str, + *, + methods: Sequence[HttpMethod] = ('GET',), + enabled: bool = True, + interactive: bool = True, + cache_control: str = 'no-store', + path_spec: JSON | None = None, + spec: JSON | None = None, + **kwargs + ) -> Callable[[C], C]: """ Decorates a view handler function in a Chalice application. @@ -286,27 +289,36 @@ def route(self, header. :param path_spec: Corresponds to an OpenAPI Paths Object. See + https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#paths-object + If multiple `@app.route` invocations refer to the same path (but with different HTTP methods), only specify this argument for one of them, otherwise an AssertionError will be raised. - :param method_spec: Corresponds to an OpenAPI Operation Object. See - https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#operation-object - This must be specified for every `@app.route` - invocation. + :param spec: Corresponds to an OpenAPI Operation Object. See + + https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.3.md#operation-object + + Even though this keyword argument has a default value, it + must be specified for every `@app.route` invocation. The + reason for the default is so that the signature of the + override is compatible with that of the overridden method, + a mypy requirement. """ + require(spec is not None, "Argument 'spec' is required") + assert spec is not None if enabled: if not interactive: require(bool(methods), 'Must list methods with interactive=False') self.non_interactive_routes.update((path, method) for method in methods) - method_spec = deep_dict_merge(method_spec, self.default_method_specs()) + spec = deep_dict_merge(spec, self.default_specs()) chalice_decorator = super().route(path, methods=methods, **kwargs) def decorator(view_func): view_func.cache_control = cache_control - self._register_spec(path, path_spec, method_spec, methods) + self._register_spec(path, methods, path_spec, spec) return chalice_decorator(view_func) return decorator @@ -326,17 +338,17 @@ def spec(self) -> JSON: Only call this method after all routes are registered. """ used_tags = set( - tag - for path in self._specs['paths'].values() - for method in path.values() if isinstance(method, dict) - for tag in method.get('tags', []) + json_str(tag) + for path in json_dict(self._specs['paths']).values() + for method in json_dict(path).values() if isinstance(method, dict) + for tag in json_list(method.get('tags', [])) ) - assert 'servers' not in self._specs + reject('servers' in self._specs, "The 'servers' entry is computed") return { **self._specs, 'tags': [ - tag for tag in self._specs.get('tags', []) - if tag['name'] in used_tags + tag for tag in json_list(self._specs.get('tags', [])) + if json_dict(tag)['name'] in used_tags ], 'servers': [{'url': str(self.base_url.add(path='/'))}] } @@ -347,7 +359,9 @@ def self_url(self) -> mutable_furl: The URL of the current request, including the path, but without query arguments. Callers can safely modify the returned `furl` instance. """ - path = self.current_request.context['path'] + request = self.current_request + assert request is not None + path = request.context['path'] return self.base_url.add(path=path) @property @@ -370,7 +384,9 @@ def base_url(self) -> mutable_furl: from chalice.constants import ( DEFAULT_HANDLER_NAME, ) - assert self.lambda_context.function_name == DEFAULT_HANDLER_NAME + lambda_context = self.lambda_context + assert lambda_context is not None + assert lambda_context.function_name == DEFAULT_HANDLER_NAME scheme = 'http' else: # Invocation via API Gateway @@ -387,25 +403,26 @@ def is_running_locally(self) -> bool: def _register_spec(self, path: str, + methods: Iterable[str], path_spec: JSON | None, - method_spec: JSON, - methods: Iterable[str]): + spec: JSON): """ Add a route's specifications to the specification object. """ + paths = json_dict(self._specs['paths']) if path_spec is not None: - assert path not in self._specs['paths'], 'Only specify path_spec once per route path' - self._specs['paths'][path] = copy_json(path_spec) + reject(path in paths, + 'Only specify path_spec once per route path') + paths[path] = copy_json(path_spec) for method in methods: # OpenAPI requires HTTP method names be lower case method = method.lower() # This may override duplicate specs from path_specs - if path not in self._specs['paths']: - self._specs['paths'][path] = {} - assert method not in self._specs['paths'][path], \ - 'Only specify method_spec once per route path and method' - self._specs['paths'][path][method] = copy_json(method_spec) + path_methods = json_dict(paths.setdefault(path, {})) + reject(method in path_methods, + "Only specify 'spec' once per route path and method") + path_methods[method] = copy_json(spec) class _LogJSONEncoder(JSONEncoder): @@ -508,10 +525,6 @@ def load_resource(self, *path: str) -> str: with open_resource(*path, package_root=package_root) as f: return f.read() - # Some type annotations to help with auto-complete - lambda_context: LambdaContext - current_request: AzulRequest - @property def catalog(self) -> str: request = self.current_request @@ -566,6 +579,7 @@ def bind(self, app: Chalice, handler_name: str | None = None) -> Self: @property def tf_function_resource_name(self) -> str: + assert self.handler_name is not None, 'Unbound decorator' if self.handler_name is None: return self.app_name else: @@ -664,7 +678,7 @@ def default_routes(self): @self.route( '/', interactive=False, - method_spec={ + spec={ 'summary': 'Redirect to the Swagger UI for interactive use of this REST API', 'tags': ['Auxiliary'], 'responses': { @@ -683,7 +697,7 @@ def swagger_redirect(): interactive=False, cache_control=self._http_cache_for(24 * 60 * 60), cors=False, - method_spec={ + spec={ 'summary': 'The Swagger UI for interactive use of this REST API', 'tags': ['Auxiliary'], 'responses': { @@ -701,7 +715,7 @@ def swagger_ui(): interactive=False, cache_control=self._http_cache_for(60), cors=True, - method_spec={ + spec={ 'summary': 'Used internally by the Swagger UI', 'tags': ['Auxiliary'], 'responses': { @@ -734,7 +748,7 @@ def swagger_initializer(): interactive=False, cache_control=self._http_cache_for(24 * 60 * 60), cors=True, - method_spec={ + spec={ 'summary': 'Static files needed for the Swagger UI', 'tags': ['Auxiliary'], 'responses': { @@ -760,7 +774,7 @@ def swagger_resource(file): methods=['GET'], cache_control=self._http_cache_for(60), cors=True, - method_spec={ + spec={ 'summary': 'Return OpenAPI specifications for this REST API', 'description': format_description(''' This endpoint returns the [OpenAPI specifications]' @@ -794,7 +808,7 @@ def openapi(): '/version', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Describe current version of this REST API', 'tags': ['Auxiliary'], 'responses': { @@ -833,7 +847,7 @@ def version(): '/robots.txt', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Robots Exclusion Protocol', 'tags': ['Auxiliary'], 'responses': { @@ -860,7 +874,7 @@ def robots_txt(): return locals() - def default_method_specs(self): + def default_specs(self): return { 'responses': { '504': { @@ -880,8 +894,10 @@ class AppController: @property def lambda_context(self) -> LambdaContext: + assert self.app.lambda_context is not None return self.app.lambda_context @property def current_request(self) -> AzulRequest: + assert self.app.current_request is not None return self.app.current_request diff --git a/src/azul/collections.py b/src/azul/collections.py index 3cc6995157..c60152cd0d 100644 --- a/src/azul/collections.py +++ b/src/azul/collections.py @@ -21,8 +21,10 @@ Any, Callable, Protocol, + Self, TypeVar, Union, + overload, ) @@ -47,7 +49,7 @@ def dict_merge(dicts: Iterable[Mapping]) -> dict: # noinspection PyPep8Naming -class deep_dict_merge: +class deep_dict_merge[K, V](dict): """ Recursively merge the given dictionaries. If more than one dictionary contains a given key, and all values associated with this key are themselves @@ -86,29 +88,29 @@ class deep_dict_merge: {} """ - def __new__(cls, *dicts: Mapping) -> dict: - return cls.from_iterable(dicts) + def __init__(self, *maps: Mapping[K, V]): + super().__init__() + self.merge(maps) @classmethod - def from_iterable(cls, dicts: Iterable[Mapping], /) -> dict: - merged = {} - for m in dicts: + def from_iterable(cls, maps: Iterable[Mapping[K, V]], /) -> Self: + self = cls() + self.merge(maps) + return self + + def merge(self, maps: Iterable[Mapping[K, V]]): + for m in maps: for k, v2 in m.items(): - v1 = merged.setdefault(k, v2) + v1 = self.setdefault(k, v2) if v1 != v2: if isinstance(v1, Mapping) and isinstance(v2, Mapping): - merged[k] = deep_dict_merge(v1, v2) + self[k] = type(self)(v1, v2) else: raise ValueError(f'{v1!r} != {v2!r}') - return merged - -K = TypeVar('K') -V = TypeVar('V') - -def explode_dict(d: Mapping[K, Union[V, list[V], set[V], tuple[V]]] - ) -> Iterable[dict[K, V]]: +def explode_dict[K, V](d: Mapping[K, Union[V, list[V], set[V], tuple[V]]] + ) -> Iterable[dict[K, V]]: """ An iterable of dictionaries, one dictionary for every possible combination of items from iterable values in the argument dictionary. Only instances of @@ -132,7 +134,7 @@ def explode_dict(d: Mapping[K, Union[V, list[V], set[V], tuple[V]]] yield dict(zip(d.keys(), t)) -def none_safe_apply(f: Callable[[K], V], o: K | None) -> V | None: +def none_safe_apply[K, V](f: Callable[[K], V], o: K | None) -> V | None: """ >>> none_safe_apply(str, 123) '123' @@ -243,10 +245,20 @@ def compose_keys(f: Callable, g: Callable) -> Callable: return lambda v: f(g(v)) -def adict(seq: Union[Mapping[K, V], Iterable[tuple[K, V]]] = None, - /, - **kwargs: V - ) -> dict[K, V]: +@overload +def adict[K, V](seq: Mapping[K, V] | Iterable[tuple[K, V]] | None = None, + / + ) -> dict[K, V]: ... + + +@overload +def adict[K, V](seq: Mapping[str, V] | Iterable[tuple[str, V]] | None = None, + /, + **kwargs: V + ) -> dict[str, V]: ... + + +def adict(seq=None, /, **kwargs): """ Like dict() but ignores keyword arguments that are None. Really only useful for literals. May be inefficient for large arguments. @@ -286,7 +298,7 @@ def _athing(cls: type, *args): return cls(arg for arg in args if arg is not None) -def atuple(*args: V) -> tuple[V, ...]: +def atuple[V](*args: V) -> tuple[V, ...]: """ >>> atuple() () @@ -300,7 +312,7 @@ def atuple(*args: V) -> tuple[V, ...]: return _athing(tuple, *args) -def alist(*args: V) -> list[V]: +def alist[V](*args: V) -> list[V]: """ >>> alist() [] @@ -314,7 +326,7 @@ def alist(*args: V) -> list[V]: return _athing(list, *args) -def aset(*args: V) -> set[V]: +def aset[V](*args: V) -> set[V]: """ >>> aset() set() @@ -379,7 +391,8 @@ def __getitem__(self, key: _KT_contra, /) -> _VT_co: ... def getitem(d: SupportsGetItem[_KT_contra, _VT_co], k: _KT_contra, /, - default: _VT_co | None = None) -> _VT_co: + default: _VT_co | None = None + ) -> _VT_co | None: """ For mappings that implement ``.__getitem__()`` but forego the recommended implementation of ``.get()``: @@ -426,7 +439,7 @@ def getitem(d: SupportsGetItem[_KT_contra, _VT_co], return default -class OrderedSet(MutableSet[K]): +class OrderedSet[K](MutableSet[K]): """ A mutable set that maintains insertion order. Unlike similar implementations of the same name floating around on the internet, it is not a sequence. @@ -487,7 +500,7 @@ def __eq__(self, other: Any) -> bool: """ return self.inner.keys() == other - def __contains__(self, member: K) -> bool: + def __contains__(self, member) -> bool: """ >>> 'a' in OrderedSet(['a', 'b']) True diff --git a/src/azul/health.py b/src/azul/health.py index c8835befcb..79974279e7 100644 --- a/src/azul/health.py +++ b/src/azul/health.py @@ -379,8 +379,8 @@ def _health_spec(health_keys: dict) -> JSON: ''') if len(health_keys) > 1 else ''), **responses.json_content( schema.object( - additional_properties=schema.object( - additional_properties=True, + additionalProperties=schema.object( + additionalProperties=True, up=schema.enum(up) ), up=schema.enum(up) @@ -398,7 +398,7 @@ def _health_spec(health_keys: dict) -> JSON: '/health', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Complete health check', 'description': format_description(f''' Health check of the {_app_name} REST API and all @@ -420,7 +420,7 @@ def health(): '/health/basic', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Basic health check', 'description': format_description(f''' Health check of only the REST API itself, excluding other @@ -438,7 +438,7 @@ def basic_health(): '/health/cached', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Cached health check for continuous monitoring', 'description': format_description(f''' Return a cached copy of the @@ -459,7 +459,7 @@ def cached_health(): '/health/fast', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Fast health check', 'description': format_description(''' Performance-optimized health check of the REST API and other @@ -479,7 +479,7 @@ def fast_health(): '/health/{keys}', methods=['GET'], cors=True, - method_spec={ + spec={ 'summary': 'Selective health check', 'description': format_description(''' This endpoint allows clients to request a health check on a @@ -492,7 +492,7 @@ def fast_health(): 'parameters': [ params.path( 'keys', - type_=schema.array(schema.enum(*sorted(Health.all_keys))), + form=schema.array(schema.enum(*sorted(Health.all_keys))), description=''' A comma-separated list of keys selecting the health checks to be performed. Each key corresponds to an diff --git a/src/azul/indexer/document.py b/src/azul/indexer/document.py index fe5430967e..eac32e54d2 100644 --- a/src/azul/indexer/document.py +++ b/src/azul/indexer/document.py @@ -676,7 +676,7 @@ def api_schema(self) -> JSON: """ The JSONSchema describing fields of this type in OpenAPI specifications. """ - return schema.make_type(self.native_type) + return schema.schema(self.native_type) def from_api(self, value: AnyJSON) -> N: """ @@ -820,7 +820,7 @@ def from_index(self, value: T) -> N: @property def api_schema(self) -> JSON: - return schema.nullable(schema.make_type(self.optional_type)) + return schema.nullable(schema.make(self.optional_type)) class NullableScalar(Nullable[N, T], metaclass=ABCMeta): @@ -828,7 +828,7 @@ class NullableScalar(Nullable[N, T], metaclass=ABCMeta): def api_filter_schema(self, relation: str) -> JSON: if relation == 'within': # The LHS operand of a range relation can't be null - api_type = schema.make_type(self.optional_type) + api_type = schema.make(self.optional_type) return self._api_range_schema(api_type) else: return super().api_filter_schema(relation) @@ -990,7 +990,7 @@ def api_filter_schema(self, relation: str) -> JSON: kwargs = dict(additionalProperties=False) if required: kwargs['required'] = required - return schema.object_type(properties, **kwargs) + return schema.object(properties=properties, **kwargs) def filter(self, relation: str, values: list[JSON]) -> list[JSON]: nested_object = one(values) diff --git a/src/azul/indexer/index_controller.py b/src/azul/indexer/index_controller.py index 0c9da8bcfc..6b16116c8a 100644 --- a/src/azul/indexer/index_controller.py +++ b/src/azul/indexer/index_controller.py @@ -16,6 +16,7 @@ import logging import time from typing import ( + Self, cast, ) import uuid @@ -76,7 +77,7 @@ class Action(Enum): delete = auto() @classmethod - def from_json(cls, action: str): + def from_json(cls, action: str) -> Self: try: return Action[action] except KeyError: @@ -85,14 +86,6 @@ def from_json(cls, action: str): def to_json(self) -> str: return self.name - def is_delete(self) -> bool: - if self is self.delete: - return True - elif self is self.add: - return False - else: - assert False - class IndexController(AppController): # The number of documents to be queued in a single SQS `send_messages`. @@ -176,7 +169,7 @@ def contribute(self, event: Iterable[SQSRecord], *, retry=False): notification = message['notification'] catalog = message['catalog'] assert catalog is not None - delete = action.is_delete() + delete = action is Action.delete contributions, replicas = self.transform(catalog, notification, delete) log.info('Writing %i contributions to index.', len(contributions)) diff --git a/src/azul/openapi/params.py b/src/azul/openapi/params.py index 97b9d42f0f..781011d703 100644 --- a/src/azul/openapi/params.py +++ b/src/azul/openapi/params.py @@ -1,13 +1,9 @@ -from typing import ( - Union, -) - from azul.openapi import ( format_description_key, schema, ) from azul.openapi.schema import ( - TYPE, + Form, ) from azul.types import ( JSON, @@ -15,7 +11,7 @@ ) -def path(name: str, type_: TYPE, **kwargs: PrimitiveJSON) -> JSON: +def path(name: str, form: Form, **kwargs: PrimitiveJSON) -> JSON: """ Returns an OpenAPI `parameters` specification of a URL path parameter. Note that path parameters cannot be optional. @@ -32,11 +28,11 @@ def path(name: str, type_: TYPE, **kwargs: PrimitiveJSON) -> JSON: } } """ - return _make_param(name, in_='path', type_=type_, **kwargs) + return _make_param(name, in_='path', form=form, **kwargs) def query(name: str, - type_: Union[TYPE, schema.optional], + form: Form | schema.optional, **kwargs: PrimitiveJSON ) -> JSON: """ @@ -54,11 +50,11 @@ def query(name: str, } } """ - return _make_param(name, in_='query', type_=type_, **kwargs) + return _make_param(name, in_='query', form=form, **kwargs) def header(name: str, - type_: Union[TYPE, schema.optional], + form: Form | schema.optional, **kwargs: PrimitiveJSON ) -> JSON: """ @@ -76,23 +72,24 @@ def header(name: str, } } """ - return _make_param(name, in_='header', type_=type_, **kwargs) + return _make_param(name, in_='header', form=form, **kwargs) def _make_param(name: str, in_: str, - type_: Union[TYPE, schema.optional], + form: Form | schema.optional, **kwargs: PrimitiveJSON ) -> JSON: - is_optional = isinstance(type_, schema.optional) - if is_optional: - type_ = type_.type_ + if isinstance(form, schema.optional): + form, required = form.form, False + else: + required = True format_description_key(kwargs) - schema_or_content = schema.make_type(type_) + schema_or_content = schema.make(form) return { 'name': name, 'in': in_, - 'required': not is_optional, + 'required': required, # https://swagger.io/docs/specification/describing-parameters/#schema-vs-content 'content' if 'application/json' in schema_or_content else 'schema': schema_or_content, **kwargs diff --git a/src/azul/openapi/responses.py b/src/azul/openapi/responses.py index a9e44254fd..b3911518e1 100644 --- a/src/azul/openapi/responses.py +++ b/src/azul/openapi/responses.py @@ -4,7 +4,7 @@ schema, ) from azul.openapi.schema import ( - TYPE, + Form, ) from azul.types import ( AnyJSON, @@ -22,7 +22,7 @@ def json_content(schema: JSON, **kwargs: AnyJSON) -> JSON: } -def header(type_: TYPE, **kwargs: PrimitiveJSON) -> JSON: +def header(form: Form, **kwargs: PrimitiveJSON) -> JSON: """ Returns the schema and description for a response header. @@ -38,6 +38,6 @@ def header(type_: TYPE, **kwargs: PrimitiveJSON) -> JSON: """ format_description_key(kwargs) return { - 'schema': schema.make_type(type_), + 'schema': schema.make(form), **kwargs } diff --git a/src/azul/openapi/schema.py b/src/azul/openapi/schema.py index 7eeabaeda8..3a54c3e592 100644 --- a/src/azul/openapi/schema.py +++ b/src/azul/openapi/schema.py @@ -1,14 +1,9 @@ -from collections.abc import ( - Mapping, -) import re from typing import ( + Mapping, NamedTuple, - Optional, - Type, - TypeVar, - Union, - get_origin, + TypeAliasType, + overload, ) from more_itertools import ( @@ -16,11 +11,14 @@ ) from azul import ( + reject, require, ) from azul.types import ( + AnyJSON, JSON, PrimitiveJSON, + reify, ) """ @@ -29,7 +27,7 @@ need of manually maintaining the `required` schema property. """ -TYPE = Union[None, Type, str, JSON] +Form = None | type | str | JSON | TypeAliasType # noinspection PyPep8Naming @@ -37,7 +35,7 @@ class optional(NamedTuple): """ Use in conjunction with `object` to mark certain properties as optional. """ - type_: TYPE + form: Form # We're consciously shadowing the `object` builtin here. Two factors mitigate @@ -46,9 +44,23 @@ class optional(NamedTuple): # wholesale and its members referenced by fully qualifying their name so the # `object` builtin is not shadowed in the importing module. +# noinspection PyShadowingBuiltins,PyPep8Naming +@overload +def object(*, + additionalProperties: JSON | bool = False, + **properties: Form | optional) -> JSON: ... + + +# noinspection PyShadowingBuiltins,PyPep8Naming +@overload +def object(*, properties: JSON, **kwargs: AnyJSON) -> JSON: ... + -# noinspection PyShadowingBuiltins -def object(additional_properties=False, **props: Union[TYPE, optional]) -> JSON: +# noinspection PyShadowingBuiltins,PyPep8Naming +def object(*, + properties=None, + additionalProperties=None, + **kwargs) -> JSON: """ >>> from azul.doctests import assert_json >>> assert_json(object(x=int, y=int, relative=optional(bool))) @@ -80,42 +92,42 @@ def object(additional_properties=False, **props: Union[TYPE, optional]) -> JSON: "properties": {}, "additionalProperties": false } - """ - new_props = {} - required = [] - for name, prop in props.items(): - if name.endswith('_'): - name = name[:-1] - if isinstance(prop, optional): - prop = prop.type_ - else: - required.append(name) - new_props[name] = prop - return object_type(properties(**new_props), - **(dict(required=required) if required else {}), - additionalProperties=additional_properties) - -def properties(**props: TYPE) -> JSON: + >>> object(x=int, y=int, relative=optional(bool)) == object( + ... properties=dict(x=dict(type="integer", format="int64"), + ... y=dict(type="integer", format="int64"), + ... relative=dict(type="boolean")), + ... additionalProperties=False, + ... required=['x','y'] + ... ) + True """ - Returns a JSON schema `properties` attribute value. - >>> from azul.doctests import assert_json - >>> assert_json(properties(x=make_type(int), y=make_type(bool))) - { - "x": { - "type": "integer", - "format": "int64" - }, - "y": { - "type": "boolean" - } + if properties is None: + properties, required = {}, [] + for name, value in kwargs.items(): + if name.endswith('_'): + name = name[:-1] + if isinstance(value, optional): + value = value.form + else: + required.append(name) + properties[name] = schema(value) + kwargs = {'required': required} if required else {} + if additionalProperties is None: + additionalProperties = False + kwargs['additionalProperties'] = additionalProperties + else: + if additionalProperties is not None: + kwargs['additionalProperties'] = additionalProperties + return { + 'type': 'object', + 'properties': properties, + **kwargs, } - """ - return {name: make_type(prop) for name, prop in props.items()} -def array(item: TYPE, *items: TYPE, **kwargs) -> JSON: +def array(item: Form, *items: Form, **kwargs) -> JSON: """ Returns the schema for an array of items of a given type, or a sequence of types. @@ -138,10 +150,10 @@ def array(item: TYPE, *items: TYPE, **kwargs) -> JSON: "additionalItems": true } """ - return array_type(make_type(item), *map(make_type, items), **kwargs) + return array_type(schema(item), *map(schema, items), **kwargs) -def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: +def enum(*items: PrimitiveJSON, form: Form = None) -> JSON: """ Returns an `enum` schema for the given items. By default, the schema type of the items is inferred, but a type may be passed explicitly to override that. @@ -149,7 +161,7 @@ def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: types of the enum values contradict the explicit type. >>> from azul.doctests import assert_json - >>> assert_json(enum('foo', 'bar', type_=str)) + >>> assert_json(enum('foo', 'bar', form=str)) { "type": "string", "enum": [ @@ -169,7 +181,7 @@ def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: ] } - >>> assert_json(enum('x', type_={'type': 'string'})) + >>> assert_json(enum('x', form={'type': 'string'})) { "type": "string", "enum": [ @@ -182,12 +194,12 @@ def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: ... ValueError: Expected exactly one item in iterable, but got , , and perhaps more. - >>> enum('foo', 'bar', type_=int) + >>> enum('foo', 'bar', form=int) Traceback (most recent call last): ... AssertionError - >>> assert_json(enum('foo', 'bar', type_="integer")) + >>> assert_json(enum('foo', 'bar', form="integer")) { "type": "integer", "enum": [ @@ -197,23 +209,23 @@ def enum(*items: PrimitiveJSON, type_: TYPE = None) -> JSON: } """ - if isinstance(type_, type): - assert all(isinstance(item, type_) for item in items) + if isinstance(form, type): + assert all(isinstance(item, form) for item in items) else: inferred_type = one(set(map(type, items))) - if type_ is None: - type_ = inferred_type + if form is None: + form = inferred_type else: # Can't easily verify type when passed as string or mapping pass return { - **make_type(type_), + **schema(form), 'enum': items } -def pattern(regex: Union[str, re.Pattern], _type: TYPE = str) -> JSON: +def pattern(regex: str | re.Pattern, _type: Form = str) -> JSON: """ Returns schema for a JSON string matching the given pattern. @@ -245,26 +257,24 @@ def pattern(regex: Union[str, re.Pattern], _type: TYPE = str) -> JSON: regex = regex.pattern assert isinstance(regex, str) return { - **make_type(_type), + **schema(_type), 'pattern': regex } -def with_default(default: PrimitiveJSON, - /, - type_: Optional[TYPE] = None - ) -> JSON: +def default(default: PrimitiveJSON, /, form: Form = None) -> JSON: """ Add a documented default value to the type schema. >>> from azul.doctests import assert_json - >>> assert_json(with_default('foo')) + + >>> assert_json(default('foo')) { "type": "string", "default": "foo" } - >>> assert_json(with_default(0, type_=float)) + >>> assert_json(default(0, form=float)) { "type": "number", "format": "double", @@ -272,22 +282,19 @@ def with_default(default: PrimitiveJSON, } """ return { - **make_type(type(default) if type_ is None else type_), + **schema(type(default) if form is None else form), 'default': default } -N = TypeVar('N', bound=Union[int, float]) - - -def in_range(minimum: Optional[N], - maximum: Optional[N], - type_: Optional[TYPE] = None - ) -> JSON: +def range[N: int | float](minimum: N | None, + maximum: N | None, + form: Form = None + ) -> JSON: """ >>> from azul.doctests import assert_json - >>> assert_json(in_range(1, 2)) + >>> assert_json(range(1, 2)) { "type": "integer", "format": "int64", @@ -295,50 +302,50 @@ def in_range(minimum: Optional[N], "maximum": 2 } - >>> assert_json(in_range(.5, None)) + >>> assert_json(range(.5, None)) { "type": "number", "format": "double", "minimum": 0.5 } - >>> assert_json(in_range(None, 2.0)) + >>> assert_json(range(None, 2.0)) { "type": "number", "format": "double", "maximum": 2.0 } - >>> assert_json(in_range(minimum=.5, maximum=2)) + >>> assert_json(range(minimum=.5, maximum=2)) Traceback (most recent call last): ... azul.RequirementError: ('Mismatched argument types', , ) - >>> assert_json(in_range()) + >>> assert_json(range()) Traceback (most recent call last): ... - TypeError: in_range() missing 2 required positional arguments: 'minimum' and 'maximum' + TypeError: range() missing 2 required positional arguments: 'minimum' and 'maximum' - >>> assert_json(in_range(None, None)) + >>> assert_json(range(None, None)) Traceback (most recent call last): ... azul.RequirementError: Must pass at least one bound """ - if type_ is None: + if form is None: types = (type(minimum), type(maximum)) set_of_types = set(types) set_of_types.discard(type(None)) require(bool(set_of_types), 'Must pass at least one bound') require(len(set_of_types) == 1, 'Mismatched argument types', *types) - type_ = one(set_of_types) + form = one(set_of_types) return { - **make_type(type_), + **schema(form), **({} if minimum is None else {'minimum': minimum}), **({} if maximum is None else {'maximum': maximum}) } -_primitive_types: Mapping[Optional[type], JSON] = { +_primitive_types: Mapping[type | None, JSON] = { str: {'type': 'string'}, bool: {'type': 'boolean'}, # Note that `format` on numeric types is an OpenAPI extension to JSONSchema @@ -357,31 +364,6 @@ def in_range(minimum: Optional[N], } -def object_type(properties: JSON, **kwargs) -> JSON: - """ - Returns the schema for a JSON object with the given properties. - - >>> from azul.doctests import assert_json - >>> assert_json(object_type({'x': {'type': 'string'}}, required=['x'])) - { - "type": "object", - "properties": { - "x": { - "type": "string" - } - }, - "required": [ - "x" - ] - } - """ - return { - 'type': 'object', - 'properties': properties, - **kwargs - } - - def array_type(item: JSON, *items: JSON, **kwargs) -> JSON: """ Returns the schema for a JSON array of items of a given type or types. @@ -410,60 +392,63 @@ def array_type(item: JSON, *items: JSON, **kwargs) -> JSON: } -def make_type(t: TYPE) -> JSON: +def schema(form: Form) -> JSON: """ Returns the schema for a Python primitive type such as `int` or a JSON schema type name such as `"boolean"`. For primitive JSON types, the corresponding Python types can be used: - >>> make_type(int) + >>> schema(int) {'type': 'integer', 'format': 'int64'} This is the most concise way of specifying a string schema: - >>> make_type(str) + >>> schema(str) {'type': 'string'} - >>> make_type(JSON) + >>> schema(JSON) {'type': 'object'} A JSON schema type name may be used instead: - >>> make_type('string') + >>> schema('string') {'type': 'string'} When a dictionary is passed, it is returned verbatim. This is useful in conjunction with the `properties` helper: - >>> make_type({'type': 'string'}) + >>> schema({'type': 'string'}) {'type': 'string'} For the JSON null schema, pass `type(None)` … - >>> make_type(type(None)) + >>> schema(type(None)) {'type': 'null'} … or just `None`. - >>> make_type(None) + >>> schema(None) {'type': 'null'} """ - if t == JSON: + if form == JSON: return {'type': 'object'} - elif t is None: - return _primitive_types[type(t)] - elif isinstance(t, type): - return _primitive_types[t] - elif isinstance(t, str): - return {'type': t} - elif isinstance(t, get_origin(JSON)): - return t + elif form is None: + return _primitive_types[type(form)] + elif isinstance(form, type): + return _primitive_types[form] + elif isinstance(form, str): + return {'type': form} + elif isinstance(form, reify(JSON)): + return form else: - assert False, type(t) + assert False, type(form) + +make = schema -def union(*ts: TYPE, for_openapi: bool = True) -> JSON: + +def union(*ts: Form, for_openapi: bool = True) -> JSON: """ The union of one or more types. @@ -486,7 +471,7 @@ def union(*ts: TYPE, for_openapi: bool = True) -> JSON: >>> union(str, int, for_openapi=False) {'anyOf': [{'type': 'string'}, {'type': 'integer', 'format': 'int64'}]} """ - ts = list(map(make_type, ts)) + ts = list(map(schema, ts)) # There are two ways to represent a union of types in JSONSchema, … if not for_openapi and all(len(t) == 1 and isinstance(t.get('type'), str) for t in ts): # … a shortcut for simple types … @@ -496,7 +481,7 @@ def union(*ts: TYPE, for_openapi: bool = True) -> JSON: return {'anyOf': ts} -def nullable(t: TYPE, for_openapi: bool = True) -> JSON: +def nullable(t: Form, for_openapi: bool = True) -> JSON: """ Given a schema, return a schema that additionally permits the `null` value. @@ -504,6 +489,8 @@ def nullable(t: TYPE, for_openapi: bool = True) -> JSON: to `optional` from this module, which is used to indicate that a property may be absent from an object. + :param t: The schema or equivalent Python type to make nullable + :param for_openapi: True to emit OpenAPI 3.0 flavor of JSONSchema, False for vanilla JSONSchema @@ -526,8 +513,8 @@ def nullable(t: TYPE, for_openapi: bool = True) -> JSON: >>> nullable(str, for_openapi=False) {'type': ['null', 'string']} """ - require(t is not None or type(None)) + reject(t is None or t is type(None)) if for_openapi: - return make_type(t) | {'nullable': True} + return {**schema(t), 'nullable': True} else: return union(None, t, for_openapi=False) diff --git a/src/azul/service/catalog_controller.py b/src/azul/service/catalog_controller.py index 10cd437d23..43c280d348 100644 --- a/src/azul/service/catalog_controller.py +++ b/src/azul/service/catalog_controller.py @@ -27,15 +27,15 @@ class CatalogController(ServiceAppController): def list_catalogs(self) -> schema.object( default_catalog=str, catalogs=schema.object( - additional_properties=schema.object( + additionalProperties=schema.object( atlas=str, internal=bool, plugins=schema.object( - additional_properties=schema.object( + additionalProperties=schema.object( name=str, sources=schema.optional(schema.array(str)), indices=schema.optional(schema.object( - additional_properties=schema.object( + additionalProperties=schema.object( default_sort=str, default_order=str ) diff --git a/src/azul/types.py b/src/azul/types.py index 0c0980c780..99a9cd0e13 100644 --- a/src/azul/types.py +++ b/src/azul/types.py @@ -8,9 +8,9 @@ from typing import ( Any, ForwardRef, - Generic, Optional, Protocol, + TypeAliasType, TypeVar, Union, get_args, @@ -27,67 +27,90 @@ # two generic types are the most specific *immutable* super-types of `list`, # `tuple` and `dict`: -AnyJSON4 = Sequence[Any] | Mapping[str | Any] | PrimitiveJSON -AnyJSON3 = Sequence[AnyJSON4] | Mapping[str, AnyJSON4] | PrimitiveJSON -AnyJSON2 = Sequence[AnyJSON3] | Mapping[str, AnyJSON3] | PrimitiveJSON -AnyJSON1 = Sequence[AnyJSON2] | Mapping[str, AnyJSON2] | PrimitiveJSON -AnyJSON = Sequence[AnyJSON1] | Mapping[str, AnyJSON1] | PrimitiveJSON -JSON = Mapping[str, AnyJSON] -JSONs = Sequence[JSON] -CompositeJSON = JSON | Sequence[AnyJSON] -FlatJSON = Mapping[str, PrimitiveJSON] +type AnyJSON = JSON | JSONArray | PrimitiveJSON +type JSON = Mapping[str, AnyJSON] +type JSONArray = Sequence[AnyJSON] +type JSONs = Sequence[JSON] +type CompositeJSON = JSON | JSONArray +type FlatJSON = Mapping[str, PrimitiveJSON] # For mutable JSON we can be more specific and use dict and list: -AnyMutableJSON4 = list[Any] | dict[str, Any] | PrimitiveJSON -AnyMutableJSON3 = list[AnyMutableJSON4] | dict[str, AnyMutableJSON4] | PrimitiveJSON -AnyMutableJSON2 = list[AnyMutableJSON3] | dict[str, AnyMutableJSON3] | PrimitiveJSON -AnyMutableJSON1 = list[AnyMutableJSON2] | dict[str, AnyMutableJSON2] | PrimitiveJSON -AnyMutableJSON = list[AnyMutableJSON1] | dict[str, AnyMutableJSON1] | PrimitiveJSON -MutableJSON = dict[str, AnyMutableJSON] -MutableJSONs = list[MutableJSON] -MutableCompositeJSON = MutableJSON | list[AnyJSON] -MutableFlatJSON = dict[str, PrimitiveJSON] +type AnyMutableJSON = MutableJSON | MutableJSONArray | PrimitiveJSON +type MutableJSON = dict[str, AnyMutableJSON] +type MutableJSONArray = list[AnyMutableJSON] +type MutableJSONs = list[MutableJSON] +type MutableCompositeJSON = MutableJSON | MutableJSONArray +type MutableFlatJSON = dict[str, PrimitiveJSON] -class LambdaContext(object): +def json_mapping(v: AnyJSON) -> JSON: + assert isinstance(v, Mapping) + return v + + +def json_sequence(v: AnyJSON) -> JSONArray: + assert isinstance(v, Sequence) + return v + + +def json_dict(v: AnyMutableJSON) -> MutableJSON: + assert isinstance(v, dict) + return v + + +def json_list(v: AnyMutableJSON) -> MutableJSONArray: + assert isinstance(v, list) + return v + + +def json_str(v: AnyMutableJSON | AnyJSON) -> str: + assert isinstance(v, str) + return v + + +def json_int(v: AnyMutableJSON | AnyJSON) -> int: + assert isinstance(v, int) + return v + + +def json_float(v: AnyMutableJSON | AnyJSON) -> float: + assert isinstance(v, float) + return v + + +def json_bool(v: AnyMutableJSON | AnyJSON) -> bool: + assert isinstance(v, bool) + return v + + +def json_none(v: AnyMutableJSON | AnyJSON) -> None: + assert v is None + return v + + +class LambdaContext: """ A stub for the AWS Lambda context """ - @property - def aws_request_id(self) -> str: - raise NotImplementedError + aws_request_id: str - @property - def log_group_name(self) -> str: - raise NotImplementedError + log_group_name: str - @property - def log_stream_name(self) -> str: - raise NotImplementedError + log_stream_name: str - @property - def function_name(self) -> str: - raise NotImplementedError + function_name: str - @property - def memory_limit_in_mb(self) -> str: - raise NotImplementedError + memory_limit_in_mb: str - @property - def function_version(self) -> str: - raise NotImplementedError + function_version: str - @property - def invoked_function_arn(self) -> str: - raise NotImplementedError + invoked_function_arn: str - def get_remaining_time_in_millis(self) -> int: - raise NotImplementedError + def get_remaining_time_in_millis(self) -> int: ... # type: ignore[empty-body] - def log(self, msg: str) -> None: - raise NotImplementedError + def log(self, msg: str) -> None: ... def is_optional(t) -> bool: @@ -127,16 +150,16 @@ def is_optional(t) -> bool: def reify(t): """ - Given a parameterized type construct, return a tuple of - subclasses of ``type`` representing all possible alternatives that can pass - for that construct at runtime. The return value is meant to be used as the - second argument to the ``isinstance`` or ``issubclass`` built-ins. + Given a parameterized type construct, return a tuple of subclasses of + ``type`` representing all possible alternatives that can pass for that + construct at runtime. The return value is meant to be used as the second + argument to the ``isinstance`` or ``issubclass`` built-ins. >>> reify(int) - + (,) >>> reify(Union[int]) - + (,) >>> reify(str | int) (, ) @@ -191,31 +214,29 @@ def reify(t): ... ValueError: ('Not a reifiable generic type', typing.Union) """ - # While `int | str` constructs a `UnionType` instance, `Union[str, int]` - # constructs an instance of `Union`, so we need to handle both. - origin = get_origin(t) - if origin in (UnionType, Union): - def f(t): + + def reify(t): + while isinstance(t, TypeAliasType): + t = t.__value__ + o = get_origin(t) + # While `int | str` constructs a `UnionType` instance, `Union[str, int]` + # constructs an instance of `Union`, so we need to handle both. + if o in (UnionType, Union): for a in get_args(t): - o = get_origin(a) - if o in (UnionType, Union): - # handle Union of Union - yield from f(a) - else: - yield a if o is None else o - - return tuple(OrderedSet(f(t))) - elif origin is not None: - return origin - elif t.__module__ == 'typing': - raise ValueError('Not a reifiable generic type', t) - else: - return t - - -def get_generic_type_params(cls: type[Generic], + yield from reify(a) + elif o is not None: + yield o + elif t.__module__ == 'typing': + raise ValueError('Not a reifiable generic type', t) + else: + yield t + + return tuple(OrderedSet(reify(t))) + + +def get_generic_type_params(cls: type, *required_types: type - ) -> Sequence[type | TypeVar | ForwardRef]: + ) -> tuple[type | TypeVar | ForwardRef, ...]: """ Inspect and validate the type parameters of a subclass of `typing.Generic`. @@ -224,9 +245,7 @@ def get_generic_type_params(cls: type[Generic], inspected class's definition. `*required_types` can be used to assert the superclasses of parameters that are types. - >>> from typing import Generic - >>> T = TypeVar(name='T') - >>> class A(Generic[T]): + >>> class A[T]: ... pass >>> class B(A[int]): ... pass @@ -234,10 +253,10 @@ def get_generic_type_params(cls: type[Generic], ... pass >>> get_generic_type_params(A) - (~T,) + (T,) >>> get_generic_type_params(A, str) - (~T,) + (T,) >>> get_generic_type_params(B) (,) diff --git a/test/test_app_logging.py b/test/test_app_logging.py index c5445830cd..26e71682ec 100644 --- a/test/test_app_logging.py +++ b/test/test_app_logging.py @@ -55,7 +55,7 @@ def test(self): app = AzulChaliceApp(__name__, '/app.py', unit_test=True, spec={}) path = '/fail/path' - @app.route(path, method_spec={}) + @app.route(path, spec={}) def fail(): raise ValueError(magic_message) diff --git a/test/test_openapi.py b/test/test_openapi.py index e37643819f..6769ebb47d 100644 --- a/test/test_openapi.py +++ b/test/test_openapi.py @@ -8,6 +8,7 @@ from azul import ( JSON, + RequirementError, ) from azul.chalice import ( AzulChaliceApp, @@ -48,13 +49,13 @@ def test_top_level_spec(self): 'Changing input object should not affect specs') def test_already_annotated_top_level_spec(self): - with self.assertRaises(AssertionError): + with self.assertRaises(RequirementError): self.app({'paths': {'/': {'already': 'annotated'}}}) def test_unannotated(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['GET', 'PUT'], method_spec={}) + @app.route('/foo', methods=['GET', 'PUT'], spec={}) def route(): pass # no coverage @@ -64,13 +65,13 @@ def route(): 'tags': [], 'servers': [{'url': 'https://fake.url/'}] } - actual_spec = self._assert_default_method_spec(app.spec()) + actual_spec = self._assert_default_spec(app.spec()) self.assertEqual(expected, actual_spec) - def test_just_method_spec(self): + def test_just_spec(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['GET', 'PUT'], method_spec={'a': 'b'}) + @app.route('/foo', methods=['GET', 'PUT'], spec={'a': 'b'}) def route(): pass # no coverage @@ -86,16 +87,16 @@ def route(): 'servers': [{'url': 'https://fake.url/'}] } - actual_spec = self._assert_default_method_spec(app.spec()) + actual_spec = self._assert_default_spec(app.spec()) self.assertEqual(expected_spec, actual_spec) - def _assert_default_method_spec(self, actual_spec: JSON) -> JSON: + def _assert_default_spec(self, actual_spec: JSON) -> JSON: actual_spec = copy_json(actual_spec) for path_spec in actual_spec['paths'].values(): - for method, method_spec in path_spec.items(): + for method, spec in path_spec.items(): methods = {'get', 'put'} # only what's used in these tests if method in methods: - responses = method_spec.pop('responses') + responses = spec.pop('responses') response = responses.pop('504') description = response.pop('description') self.assertIn('Request timed out', description) @@ -109,17 +110,27 @@ def test_fully_annotated_override(self): 'get': {'c': 'd'} } - with self.assertRaises(AssertionError) as cm: - @app.route('/foo', methods=['GET'], path_spec=path_spec, method_spec={'e': 'f'}) + with self.assertRaises(RequirementError) as cm: + @app.route('/foo', + methods=['GET'], + path_spec=path_spec, + spec={'e': 'f'}) def route(): pass # no coverage - self.assertEqual(str(cm.exception), 'Only specify method_spec once per route path and method') + self.assertEqual(str(cm.exception), + "Only specify 'spec' once per route path and method") def test_multiple_routes(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['GET', 'PUT'], path_spec={'a': 'b'}, method_spec={'c': 'd'}) - @app.route('/foo/too', methods=['GET'], path_spec={'e': 'f'}, method_spec={'g': 'h'}) + @app.route('/foo', + methods=['GET', 'PUT'], + path_spec={'a': 'b'}, + spec={'c': 'd'}) + @app.route('/foo/too', + methods=['GET'], + path_spec={'e': 'f'}, + spec={'g': 'h'}) def route(): pass # no coverage @@ -139,31 +150,33 @@ def route(): 'tags': [], 'servers': [{'url': 'https://fake.url/'}] } - actual_spec = self._assert_default_method_spec(app.spec()) + actual_spec = self._assert_default_spec(app.spec()) self.assertEqual(expected_specs, actual_spec) - def test_duplicate_method_specs(self): + def test_duplicate_specs(self): app = self.app({'foo': 'bar'}) - with self.assertRaises(AssertionError) as cm: - @app.route('/foo', methods=['GET'], method_spec={'a': 'b'}) - @app.route('/foo', methods=['GET'], method_spec={'a': 'XXX'}) + with self.assertRaises(RequirementError) as cm: + @app.route('/foo', methods=['GET'], spec={'a': 'b'}) + @app.route('/foo', methods=['GET'], spec={'a': 'XXX'}) def route(): pass - self.assertEqual(str(cm.exception), 'Only specify method_spec once per route path and method') + self.assertEqual("Only specify 'spec' once per route path and method", + str(cm.exception)) def test_duplicate_path_specs(self): app = self.app({'foo': 'bar'}) - @app.route('/foo', methods=['PUT'], path_spec={'a': 'XXX'}, method_spec={}) + @app.route('/foo', methods=['PUT'], path_spec={'a': 'XXX'}, spec={}) def route1(): pass - with self.assertRaises(AssertionError) as cm: - @app.route('/foo', methods=['GET'], path_spec={'a': 'b'}, method_spec={}) + with self.assertRaises(RequirementError) as cm: + @app.route('/foo', methods=['GET'], path_spec={'a': 'b'}, spec={}) def route2(): pass - self.assertEqual(str(cm.exception), 'Only specify path_spec once per route path') + self.assertEqual('Only specify path_spec once per route path', + str(cm.exception)) def test_shared_path_spec(self): """ @@ -181,19 +194,19 @@ def test_shared_path_spec(self): methods=['GET'], cors=True, path_spec=shared_path_spec, - method_spec={'summary': f'Swagger test {i}'}) + spec={'summary': f'Swagger test {i}'}) def swagger_test(): pass - method_specs = app.spec()['paths'].values() - self.assertNotEqual(*method_specs) + specs = app.spec()['paths'].values() + self.assertNotEqual(*specs) def test_unused_tags(self): app = self.app({ 'tags': [{'name': name} for name in ('foo', 'bar', 'baz', 'qux')] }) - @app.route('/foo', methods=['PUT'], method_spec={'tags': ['foo', 'qux']}) + @app.route('/foo', methods=['PUT'], spec={'tags': ['foo', 'qux']}) def route1(): pass @@ -277,7 +290,7 @@ def test_misuse(self): # wrapper. try: # noinspection PyTypeChecker - schema.make_type(schema.optional(str)) + schema.make(schema.optional(str)) except AssertionError as e: self.assertIn(schema.optional, e.args) else: