From e218cc736d858228bcca4494f8cee5cf4a198f2d Mon Sep 17 00:00:00 2001 From: Stanislav Khlud Date: Fri, 10 Jan 2025 16:40:40 +0700 Subject: [PATCH] Set up export/import action mixin --- docs/api_drf.rst | 9 + docs/getting_started.rst | 14 ++ import_export_extensions/api/__init__.py | 4 +- .../api/mixins/__init__.py | 2 + .../api/{mixins.py => mixins/common.py} | 0 import_export_extensions/api/mixins/export.py | 126 ++++++++++++ .../api/serializers/export_job.py | 12 +- .../api/views/__init__.py | 2 + .../api/views/export_job.py | 194 ++++++------------ test_project/fake_app/api/views.py | 45 +++- .../integration_tests/test_api/test_export.py | 106 +++++++++- test_project/urls.py | 12 ++ 12 files changed, 377 insertions(+), 149 deletions(-) create mode 100644 import_export_extensions/api/mixins/__init__.py rename import_export_extensions/api/{mixins.py => mixins/common.py} (100%) create mode 100644 import_export_extensions/api/mixins/export.py diff --git a/docs/api_drf.rst b/docs/api_drf.rst index 00b8bdb..445164f 100644 --- a/docs/api_drf.rst +++ b/docs/api_drf.rst @@ -14,9 +14,18 @@ API (Rest Framework) .. autoclass:: import_export_extensions.api.ExportJobForUserViewSet :members: +.. autoclass:: import_export_extensions.api.BaseExportJobViewSet + :members: + +.. autoclass:: import_export_extensions.api.BaseExportJobForUserViewSet + :members: + .. autoclass:: import_export_extensions.api.LimitQuerySetToCurrentUserMixin :members: +.. autoclass:: import_export_extensions.api.ExportStartActionMixin + :members: + .. autoclass:: import_export_extensions.api.CreateExportJob :members: create, validate diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 38a1931..e916dd6 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -169,3 +169,17 @@ the OpenAPI specification will be available. .. figure:: _static/images/bands-openapi.png A screenshot of the generated OpenAPI specification + + +Import/Export API actions mixins +----------------- + +Alternatively you can use ``api.mixins.ExportStartActionMixin`` and ``api.mixins.ImportStartActionMixin`` +to add to your current viewsets to create import/export jobs. +You would also need to use ``api.views.BaseExportJobViewSet/BaseExportJobForUsersViewSet`` +and ``api.views.BaseExportJobViewSet/BaseImportJobForUsersViewSet`` to setup endpoints to be able to: + +* ``list`` - Returns a list of jobs for the ``resource_class`` set in ViewSet +* ``retrieve`` - Returns details of a job based on the provided ID +* ``cancel`` - Stops the import/export process and sets the job's status to ``CANCELLED``. +* ``confirm`` - Confirms the import after the parse stage. This action is available only in import jobs. diff --git a/import_export_extensions/api/__init__.py b/import_export_extensions/api/__init__.py index 392cb62..8b8339b 100644 --- a/import_export_extensions/api/__init__.py +++ b/import_export_extensions/api/__init__.py @@ -1,4 +1,4 @@ -from .mixins import LimitQuerySetToCurrentUserMixin +from .mixins import ExportStartActionMixin, LimitQuerySetToCurrentUserMixin from .serializers import ( CreateExportJob, CreateImportJob, @@ -8,6 +8,8 @@ ProgressSerializer, ) from .views import ( + BaseExportJobForUserViewSet, + BaseExportJobViewSet, ExportJobForUserViewSet, ExportJobViewSet, ImportJobForUserViewSet, diff --git a/import_export_extensions/api/mixins/__init__.py b/import_export_extensions/api/mixins/__init__.py new file mode 100644 index 0000000..0dfb00e --- /dev/null +++ b/import_export_extensions/api/mixins/__init__.py @@ -0,0 +1,2 @@ +from .common import LimitQuerySetToCurrentUserMixin +from .export import ExportStartActionMixin diff --git a/import_export_extensions/api/mixins.py b/import_export_extensions/api/mixins/common.py similarity index 100% rename from import_export_extensions/api/mixins.py rename to import_export_extensions/api/mixins/common.py diff --git a/import_export_extensions/api/mixins/export.py b/import_export_extensions/api/mixins/export.py new file mode 100644 index 0000000..2c091a9 --- /dev/null +++ b/import_export_extensions/api/mixins/export.py @@ -0,0 +1,126 @@ +import collections.abc +import contextlib +import typing + +from django.conf import settings +from django.utils import module_loading + +from rest_framework import ( + decorators, + request, + response, + status, +) + +from ... import resources +from .. import serializers + + +class ExportStartActionMixin: + """Mixin which adds start export action.""" + + resource_class: type[resources.CeleryModelResource] + export_action = "export" + export_detail_serializer_class = serializers.ExportJobSerializer + export_ordering: collections.abc.Sequence[str] = () + export_ordering_fields: collections.abc.Sequence[str] = () + + def __init_subclass__(cls) -> None: + super().__init_subclass__() + # Skip if it is has no resource_class specified + if not hasattr(cls, "resource_class"): + return + filter_backends = [ + module_loading.import_string( + settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND, + ), + ] + if cls.export_ordering_fields: + filter_backends.append( + module_loading.import_string( + settings.DRF_EXPORT_ORDERING_BACKEND, + ), + ) + decorators.action( + methods=["POST"], + detail=False, + queryset=cls.resource_class.get_model_queryset(), + serializer_class=cls().get_export_create_serializer_class(), + filterset_class=getattr( + cls.resource_class, + "filterset_class", + None, + ), + filter_backends=filter_backends, + ordering=cls.export_ordering, + ordering_fields=cls.export_ordering_fields, + )(getattr(cls, cls.export_action)) + # Correct specs of drf-spectacular if it is installed + with contextlib.suppress(ImportError): + from drf_spectacular import utils + + utils.extend_schema_view( + **{ + cls.export_action: utils.extend_schema( + filters=True, + responses={ + status.HTTP_201_CREATED: cls().get_export_detail_serializer_class(), # noqa: E501 + }, + ), + }, + )(cls) + + def get_queryset(self): + """Return export model queryset on export action. + + For better openapi support and consistency. + + """ + if self.action == self.export_action: + return self.resource_class.get_model_queryset() # pragma: no cover + return super().get_queryset() + + def get_export_detail_serializer_class(self): + """Get serializer which will be used show details of export job.""" + return self.export_detail_serializer_class + + def get_export_create_serializer_class(self): + """Get serializer which will be used to start export job.""" + return serializers.get_create_export_job_serializer( + self.resource_class, + ) + + def get_export_resource_kwargs(self) -> dict[str, typing.Any]: + """Provide extra arguments to resource class.""" + return {} + + def get_serializer(self, *args, **kwargs): + """Provide resource kwargs to serializer class.""" + if self.action == self.export_action: + kwargs.setdefault( + "resource_kwargs", + self.get_export_resource_kwargs(), + ) + return super().get_serializer(*args, **kwargs) + + def export(self, request: request.Request) -> response.Response: + """Start export job.""" + return self.start_export(request) + + def start_export(self, request: request.Request) -> response.Response: + """Validate request data and start ExportJob.""" + query_params = dict(request.query_params) + ordering = query_params.pop("ordering", self.ordering) + serializer = self.get_serializer( + data=request.data, + ordering=ordering, + filter_kwargs=query_params, + ) + serializer.is_valid(raise_exception=True) + export_job = serializer.save() + return response.Response( + data=self.get_export_detail_serializer_class()( + instance=export_job, + ).data, + status=status.HTTP_201_CREATED, + ) diff --git a/import_export_extensions/api/serializers/export_job.py b/import_export_extensions/api/serializers/export_job.py index 8b26b13..19827f1 100644 --- a/import_export_extensions/api/serializers/export_job.py +++ b/import_export_extensions/api/serializers/export_job.py @@ -98,10 +98,19 @@ def update(self, instance, validated_data): """Empty method to pass linters checks.""" +# Use it to cache already generated serializers to avoid duplication +_GENERATED_EXPORT_JOB_SERIALIZERS: dict[ + type[resources.CeleryModelResource], + type, +] = {} + + def get_create_export_job_serializer( resource: type[resources.CeleryModelResource], ) -> type: """Create serializer for ExportJobs creation.""" + if resource in _GENERATED_EXPORT_JOB_SERIALIZERS: + return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] class _CreateExportJob(CreateExportJob): """Serializer to start export job.""" @@ -115,8 +124,9 @@ class _CreateExportJob(CreateExportJob): ], ) - return type( + _GENERATED_EXPORT_JOB_SERIALIZERS[resource] = type( f"{resource.__name__}CreateExportJob", (_CreateExportJob,), {}, ) + return _GENERATED_EXPORT_JOB_SERIALIZERS[resource] diff --git a/import_export_extensions/api/views/__init__.py b/import_export_extensions/api/views/__init__.py index 2409e38..5d50ae2 100644 --- a/import_export_extensions/api/views/__init__.py +++ b/import_export_extensions/api/views/__init__.py @@ -1,4 +1,6 @@ from .export_job import ( + BaseExportJobForUserViewSet, + BaseExportJobViewSet, ExportJobForUserViewSet, ExportJobViewSet, ) diff --git a/import_export_extensions/api/views/export_job.py b/import_export_extensions/api/views/export_job.py index a2d3e9e..383297f 100644 --- a/import_export_extensions/api/views/export_job.py +++ b/import_export_extensions/api/views/export_job.py @@ -1,100 +1,90 @@ import collections.abc import contextlib -import typing - -from django.conf import settings -from django.utils import module_loading from rest_framework import ( decorators, exceptions, mixins, permissions, + request, response, status, viewsets, ) -from rest_framework.request import Request import django_filters -from ... import models, resources +from ... import models from .. import mixins as core_mixins -from .. import serializers -class ExportBase(type): - """Add custom create action for each ExportJobViewSet.""" +class BaseExportJobViewSet( + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Base viewset for managing export jobs.""" - def __new__(cls, name, bases, attrs, **kwargs): - """Dynamically create an export start api endpoint. + permission_classes = (permissions.IsAuthenticated,) + serializer_class = core_mixins.ExportStartActionMixin.export_detail_serializer_class # noqa: E501 + queryset = models.ExportJob.objects.all() + filterset_class: django_filters.rest_framework.FilterSet | None = None + search_fields: collections.abc.Sequence[str] = ("id",) + ordering: collections.abc.Sequence[str] = ( + "id", + ) + ordering_fields: collections.abc.Sequence[str] = ( + "id", + "created", + "modified", + ) - We need this to specify on fly action's filterset_class and queryset - (django-filters requires view's queryset and filterset_class's - queryset model to match). Also, if drf-spectacular is installed - specify request and response, and enable filters. + def __init_subclass__(cls) -> None: + """Dynamically create an cancel api endpoints. + + Need to do this to enable action and correct open-api spec generated by + drf_spectacular. """ - viewset: type[ExportJobViewSet] = super().__new__( - cls, - name, - bases, - attrs, - **kwargs, - ) - # Skip if it is has no resource_class specified - if not hasattr(viewset, "resource_class"): - return viewset - filter_backends = [ - module_loading.import_string(settings.DRF_EXPORT_DJANGO_FILTERS_BACKEND), - ] - if viewset.export_ordering_fields: - filter_backends.append( - module_loading.import_string(settings.DRF_EXPORT_ORDERING_BACKEND), - ) - decorators.action( - methods=["POST"], - detail=False, - queryset=viewset.resource_class.get_model_queryset(), - filterset_class=getattr( - viewset.resource_class, "filterset_class", None, - ), - filter_backends=filter_backends, - ordering=viewset.export_ordering, - ordering_fields=viewset.export_ordering_fields, - )(viewset.start) + super().__init_subclass__() decorators.action( methods=["POST"], detail=True, - )(viewset.cancel) + )(cls.cancel) # Correct specs of drf-spectacular if it is installed with contextlib.suppress(ImportError): from drf_spectacular.utils import extend_schema, extend_schema_view - - detail_serializer_class = viewset().get_detail_serializer_class() - return extend_schema_view( - start=extend_schema( - filters=True, - request=viewset().get_export_create_serializer_class(), - responses={ - status.HTTP_201_CREATED: detail_serializer_class, - }, - ), + if hasattr(cls, "get_export_detail_serializer_class"): + response_serializer = cls().get_export_detail_serializer_class() # noqa: E501 + else: + response_serializer = cls().get_serializer_class() + extend_schema_view( cancel=extend_schema( request=None, responses={ - status.HTTP_200_OK: detail_serializer_class, + status.HTTP_200_OK: response_serializer, }, ), - )(viewset) - return viewset + )(cls) + def cancel(self, *args, **kwargs) -> response.Response: + """Cancel export job that is in progress.""" + job: models.ExportJob = self.get_object() + + try: + job.cancel_export() + except ValueError as error: + raise exceptions.ValidationError(error.args[0]) from error + + serializer = self.get_serializer(instance=job) + return response.Response( + status=status.HTTP_200_OK, + data=serializer.data, + ) class ExportJobViewSet( - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - viewsets.GenericViewSet, - metaclass=ExportBase, + core_mixins.ExportStartActionMixin, + BaseExportJobViewSet, ): """Base API viewset for ExportJob model. @@ -106,22 +96,7 @@ class ExportJobViewSet( """ - permission_classes = (permissions.IsAuthenticated,) - queryset = models.ExportJob.objects.all() - serializer_class = serializers.ExportJobSerializer - resource_class: type[resources.CeleryModelResource] - filterset_class: django_filters.rest_framework.FilterSet | None = None - search_fields: collections.abc.Sequence[str] = ("id",) - ordering: collections.abc.Sequence[str] = ( - "id", - ) - ordering_fields: collections.abc.Sequence[str] = ( - "id", - "created", - "modified", - ) - export_ordering: collections.abc.Sequence[str] = () - export_ordering_fields: collections.abc.Sequence[str] = () + export_action = "start" def get_queryset(self): """Filter export jobs by resource used in viewset.""" @@ -129,64 +104,9 @@ def get_queryset(self): resource_path=self.resource_class.class_path, ) - def get_resource_kwargs(self) -> dict[str, typing.Any]: - """Provide extra arguments to resource class.""" - return {} - - def get_serializer(self, *args, **kwargs): - """Provide resource kwargs to serializer class.""" - if self.action == "start": - kwargs.setdefault("resource_kwargs", self.get_resource_kwargs()) - return super().get_serializer(*args, **kwargs) - - def get_serializer_class(self): - """Return special serializer on creation.""" - if self.action == "start": - return self.get_export_create_serializer_class() - return self.get_detail_serializer_class() - - def get_detail_serializer_class(self): - """Get serializer which will be used show details of export job.""" - return self.serializer_class - - def get_export_create_serializer_class(self): - """Get serializer which will be used to start export job.""" - return serializers.get_create_export_job_serializer( - self.resource_class, - ) - - def start(self, request: Request): + def start(self, request: request.Request) -> response.Response: """Validate request data and start ExportJob.""" - query_params = dict(request.query_params) - ordering = query_params.pop("ordering", self.ordering) - serializer = self.get_serializer( - data=request.data, - ordering=ordering, - filter_kwargs=query_params, - ) - serializer.is_valid(raise_exception=True) - export_job = serializer.save() - return response.Response( - data=self.get_detail_serializer_class()( - instance=export_job, - ).data, - status=status.HTTP_201_CREATED, - ) - - def cancel(self, *args, **kwargs): - """Cancel export job that is in progress.""" - job: models.ExportJob = self.get_object() - - try: - job.cancel_export() - except ValueError as error: - raise exceptions.ValidationError(error.args[0]) from error - - serializer = self.get_serializer(instance=job) - return response.Response( - status=status.HTTP_200_OK, - data=serializer.data, - ) + return self.start_export(request) class ExportJobForUserViewSet( @@ -194,3 +114,9 @@ class ExportJobForUserViewSet( ExportJobViewSet, ): """Viewset for providing export feature to users.""" + +class BaseExportJobForUserViewSet( + core_mixins.LimitQuerySetToCurrentUserMixin, + BaseExportJobViewSet, +): + """Viewset for providing export job management to users.""" diff --git a/test_project/fake_app/api/views.py b/test_project/fake_app/api/views.py index a5e860e..1698c03 100644 --- a/test_project/fake_app/api/views.py +++ b/test_project/fake_app/api/views.py @@ -1,18 +1,51 @@ -from import_export_extensions.api import views +from rest_framework import mixins, serializers, viewsets -from ..resources import SimpleArtistResource +from import_export_extensions import api +from .. import models, resources -class ArtistExportViewSet(views.ExportJobForUserViewSet): + +class ArtistExportViewSet(api.ExportJobForUserViewSet): """Simple ViewSet for exporting Artist model.""" - resource_class = SimpleArtistResource + resource_class = resources.SimpleArtistResource export_ordering_fields = ( "id", "name", ) -class ArtistImportViewSet(views.ImportJobForUserViewSet): +class ArtistImportViewSet(api.ImportJobForUserViewSet): """Simple ViewSet for importing Artist model.""" - resource_class = SimpleArtistResource + resource_class = resources.SimpleArtistResource + +class ArtistSerializer(serializers.ModelSerializer): + """Serializer for Artist model.""" + + class Meta: + model = models.Artist + fields = ( + "id", + "name", + "instrument", + ) + +class ArtistViewSet( + api.ExportStartActionMixin, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + viewsets.GenericViewSet, +): + """Simple viewset for Artist model.""" + + resource_class = resources.SimpleArtistResource + queryset = models.Artist.objects.all() + serializer_class = ArtistSerializer + filterset_class = resources.SimpleArtistResource.filterset_class + ordering = ( + "id", + ) + ordering_fields = ( + "id", + "name", + ) diff --git a/test_project/tests/integration_tests/test_api/test_export.py b/test_project/tests/integration_tests/test_api/test_export.py index 7a23afc..c4c4920 100644 --- a/test_project/tests/integration_tests/test_api/test_export.py +++ b/test_project/tests/integration_tests/test_api/test_export.py @@ -20,6 +20,14 @@ f"{reverse('export-artist-start')}?name=Artist", id="Url with valid filter_kwargs", ), + pytest.param( + f"{reverse('artists-export')}", + id="Action url without filter_kwargs", + ), + pytest.param( + f"{reverse('artists-export')}?name=Artist", + id="Action url with valid filter_kwargs", + ), ], ) def test_export_api_creates_export_job( @@ -39,12 +47,26 @@ def test_export_api_creates_export_job( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + f"{reverse('export-artist-start')}?id=invalid_id", + id="Url with invalid filter_kwargs", + ), + pytest.param( + f"{reverse('artists-export')}?id=invalid_i", + id="Action url with invalid filter_kwargs", + ), + ], +) def test_export_api_create_export_job_with_invalid_filter_kwargs( admin_api_client: test.APIClient, + export_url: str, ): """Ensure export start API with invalid kwargs return an error.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?id=invalid_id", + path=export_url, data={ "file_format": "csv", }, @@ -54,12 +76,26 @@ def test_export_api_create_export_job_with_invalid_filter_kwargs( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + f"{reverse('export-artist-start')}?ordering=invalid_id", + id="Url with invalid filter_kwargs", + ), + pytest.param( + f"{reverse('artists-export')}?ordering=invalid_id", + id="Action url with invalid filter_kwargs", + ), + ], +) def test_export_api_create_export_job_with_invalid_ordering( admin_api_client: test.APIClient, + export_url: str, ): """Ensure export start API with invalid ordering return an error.""" response = admin_api_client.post( - path=f"{reverse('export-artist-start')}?ordering=invalid_id", + path=export_url, data={ "file_format": "csv", }, @@ -72,14 +108,28 @@ def test_export_api_create_export_job_with_invalid_ordering( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + "export-artist-detail", + id="Usual url", + ), + pytest.param( + "export-jobs-detail", + id="Action url", + ), + ], +) def test_export_api_detail( admin_api_client: test.APIClient, artist_export_job: ExportJob, + export_url: str, ): """Ensure export detail API shows current export job status.""" response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -88,15 +138,29 @@ def test_export_api_detail( @pytest.mark.django_db(transaction=True) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + "export-artist-detail", + id="Usual url", + ), + pytest.param( + "export-jobs-detail", + id="Action url", + ), + ], +) def test_import_user_api_get_detail( user: User, admin_api_client: test.APIClient, artist_export_job: ExportJob, + export_url: str, ): """Ensure import detail api for user returns only users jobs.""" response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -106,7 +170,7 @@ def test_import_user_api_get_detail( artist_export_job.save() response = admin_api_client.get( path=reverse( - "export-artist-detail", + export_url, kwargs={"pk": artist_export_job.id}, ), ) @@ -121,17 +185,31 @@ def test_import_user_api_get_detail( ExportJob.ExportStatus.EXPORTING, ], ) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + "export-artist-cancel", + id="Usual url", + ), + pytest.param( + "export-jobs-cancel", + id="Action url", + ), + ], +) def test_export_api_cancel( admin_api_client: test.APIClient, artist_export_job: ExportJob, allowed_cancel_status: ExportJob.ExportStatus, + export_url: str, ): """Ensure that export canceled with allowed statuses.""" artist_export_job.export_status = allowed_cancel_status artist_export_job.save() response = admin_api_client.post( path=reverse( - "export-artist-cancel", + export_url, kwargs={"pk": artist_export_job.pk}, ), ) @@ -149,17 +227,31 @@ def test_export_api_cancel( ExportJob.ExportStatus.CANCELLED, ], ) +@pytest.mark.parametrize( + argnames=["export_url"], + argvalues=[ + pytest.param( + "export-artist-cancel", + id="Usual url", + ), + pytest.param( + "export-jobs-cancel", + id="Action url", + ), + ], +) def test_export_api_cancel_with_errors( admin_api_client: test.APIClient, artist_export_job: ExportJob, incorrect_job_status: ExportJob.ExportStatus, + export_url: str, ): """Ensure that export job with incorrect statuses cannot be canceled.""" artist_export_job.export_status = incorrect_job_status artist_export_job.save() response = admin_api_client.post( path=reverse( - "export-artist-cancel", + export_url, kwargs={"pk": artist_export_job.pk}, ), ) diff --git a/test_project/urls.py b/test_project/urls.py index 413e23c..05d69ea 100644 --- a/test_project/urls.py +++ b/test_project/urls.py @@ -7,6 +7,8 @@ from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView +from import_export_extensions import api + from .fake_app.api import views ie_router = DefaultRouter() @@ -15,11 +17,21 @@ views.ArtistExportViewSet, basename="export-artist", ) +ie_router.register( + "export-jobs", + api.BaseExportJobForUserViewSet, + basename="export-jobs", +) ie_router.register( "import-artist", views.ArtistImportViewSet, basename="import-artist", ) +ie_router.register( + "artists", + views.ArtistViewSet, + basename="artists", +) urlpatterns = [re_path("^admin/", admin.site.urls), *ie_router.urls]