Merge pull request #942 from jobselko/fix_933

[PULP-731] Add synchronous upload API

Author: gerrod3

CHANGES/933.feature

@@ -0,0 +1 @@
+Added a synchronous upload API.

pulp_python/app/migrations/0015_alter_pythonpackagecontent_options.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.23 on 2025-08-19 17:10
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("python", "0014_pythonpackagecontent_dynamic_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="pythonpackagecontent",
+            options={
+                "default_related_name": "%(app_label)s_%(model_name)s",
+                "permissions": [
+                    ("upload_python_packages", "Can upload Python packages using synchronous API.")
+                ],
+            },
+        ),
+    ]

pulp_python/app/models.py

@@ -227,6 +227,9 @@ def __str__(self):
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"
         unique_together = ("sha256", "_pulp_domain")
+        permissions = [
+            ("upload_python_packages", "Can upload Python packages using synchronous API."),
+        ]
 
 
 class PythonPublication(Publication, AutoAddObjPermsMixin):

pulp_python/app/serializers.py

@@ -1,5 +1,8 @@
+import logging
+import os
 from gettext import gettext as _
 from django.conf import settings
+from django.db.utils import IntegrityError
 from packaging.requirements import Requirement
 from rest_framework import serializers
 
@@ -8,7 +11,15 @@
 from pulpcore.plugin.util import get_domain
 
 from pulp_python.app import models as python_models
-from pulp_python.app.utils import artifact_to_python_content_data
+from pulp_python.app.utils import (
+    DIST_EXTENSIONS,
+    artifact_to_python_content_data,
+    get_project_metadata_from_file,
+    parse_project_metadata,
+)
+
+
+log = logging.getLogger(__name__)
 
 
 class PythonRepositorySerializer(core_serializers.RepositorySerializer):
@@ -207,15 +218,15 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
         required=False,
         allow_blank=True,
         help_text=_(
-            "The Python version(s) that the distribution is guaranteed to be " "compatible with."
+            "The Python version(s) that the distribution is guaranteed to be compatible with."
         ),
     )
     # Version 2.1
     description_content_type = serializers.CharField(
         required=False,
         allow_blank=True,
         help_text=_(
-            "A string stating the markup syntax (if any) used in the distribution’s"
+            "A string stating the markup syntax (if any) used in the distribution's"
             " description, so that tools can intelligently render the description."
         ),
     )
@@ -256,7 +267,7 @@ class PythonPackageContentSerializer(core_serializers.SingleArtifactContentUploa
     )
     packagetype = serializers.CharField(
         help_text=_(
-            "The type of the distribution package " "(e.g. sdist, bdist_wheel, bdist_egg, etc)"
+            "The type of the distribution package (e.g. sdist, bdist_wheel, bdist_egg, etc)"
         ),
         read_only=True,
     )
@@ -357,6 +368,70 @@ class Meta:
         model = python_models.PythonPackageContent
 
 
+class PythonPackageContentUploadSerializer(PythonPackageContentSerializer):
+    """
+    A serializer for requests to synchronously upload Python packages.
+    """
+
+    def validate(self, data):
+        """
+        Validates an uploaded Python package file, extracts its metadata,
+        and creates or retrieves an associated Artifact.
+
+        Returns updated data with artifact and metadata details.
+        """
+        file = data.pop("file")
+        filename = file.name
+
+        for ext, packagetype in DIST_EXTENSIONS.items():
+            if filename.endswith(ext):
+                break
+        else:
+            raise serializers.ValidationError(
+                _(
+                    "Extension on {} is not a valid python extension "
+                    "(.whl, .exe, .egg, .tar.gz, .tar.bz2, .zip)"
+                ).format(filename)
+            )
+
+        # Replace the incorrect file name in the file path with the original file name
+        original_filepath = file.file.name
+        path_to_file, tmp_str = original_filepath.rsplit("/", maxsplit=1)
+        tmp_str = tmp_str.split(".", maxsplit=1)[0]  # Remove e.g. ".upload.gz" suffix
+        new_filepath = f"{path_to_file}/{tmp_str}{filename}"
+        os.rename(original_filepath, new_filepath)
+
+        metadata = get_project_metadata_from_file(new_filepath)
+        artifact = core_models.Artifact.init_and_validate(new_filepath)
+        try:
+            artifact.save()
+        except IntegrityError:
+            artifact = core_models.Artifact.objects.get(
+                sha256=artifact.sha256, pulp_domain=get_domain()
+            )
+            artifact.touch()
+            log.info(f"Artifact for {file.name} already existed in database")
+
+        data["artifact"] = artifact
+        data["sha256"] = artifact.sha256
+        data["relative_path"] = filename
+        data.update(parse_project_metadata(vars(metadata)))
+        # Overwrite filename from metadata
+        data["filename"] = filename
+        return data
+
+    class Meta(PythonPackageContentSerializer.Meta):
+        # This API does not support uploading to a repository or using a custom relative_path
+        fields = tuple(
+            f
+            for f in PythonPackageContentSerializer.Meta.fields
+            if f not in ["repository", "relative_path"]
+        )
+        model = python_models.PythonPackageContent
+        # Name used for the OpenAPI request object
+        ref_name = "PythonPackageContentUpload"
+
+
 class MinimalPythonPackageContentSerializer(PythonPackageContentSerializer):
     """
     A Serializer for PythonPackageContent.
@@ -503,7 +578,7 @@ class PythonPublicationSerializer(core_serializers.PublicationSerializer):
 
     distributions = core_serializers.DetailRelatedField(
         help_text=_(
-            "This publication is currently being hosted as configured by these " "distributions."
+            "This publication is currently being hosted as configured by these distributions."
         ),
         source="distribution_set",
         view_name="pythondistributions-detail",

pulp_python/app/viewsets.py

@@ -1,4 +1,5 @@
 from bandersnatch.configuration import BandersnatchConfig
+from django.db import transaction
 from drf_spectacular.utils import extend_schema
 from rest_framework import status
 from rest_framework.decorators import action
@@ -355,10 +356,49 @@ class PythonPackageSingleArtifactContentUploadViewSet(
                     "has_upload_param_model_or_domain_or_obj_perms:core.change_upload",
                 ],
             },
+            {
+                "action": ["upload"],
+                "principal": "authenticated",
+                "effect": "allow",
+                "condition": [
+                    "has_model_or_domain_perms:python.upload_python_packages",
+                ],
+            },
         ],
         "queryset_scoping": {"function": "scope_queryset"},
     }
 
+    LOCKED_ROLES = {
+        "python.python_package_uploader": [
+            "python.upload_python_packages",
+        ],
+    }
+
+    @extend_schema(
+        summary="Synchronous Python package upload",
+        request=python_serializers.PythonPackageContentUploadSerializer,
+        responses={201: python_serializers.PythonPackageContentSerializer},
+    )
+    @action(
+        detail=False,
+        methods=["post"],
+        serializer_class=python_serializers.PythonPackageContentUploadSerializer,
+    )
+    def upload(self, request):
+        """
+        Create a Python package.
+        """
+        serializer = self.get_serializer(data=request.data)
+
+        with transaction.atomic():
+            # Create the artifact
+            serializer.is_valid(raise_exception=True)
+            # Create the package
+            serializer.save()
+
+        headers = self.get_success_headers(serializer.data)
+        return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
+
 
 class PythonRemoteViewSet(core_viewsets.RemoteViewSet, core_viewsets.RolesMixin):
     """

pulp_python/tests/functional/api/test_upload.py

@@ -0,0 +1,44 @@
+import pytest
+from pulp_python.tests.functional.constants import (
+    PYTHON_EGG_FILENAME,
+    PYTHON_EGG_URL,
+    PYTHON_WHEEL_FILENAME,
+    PYTHON_WHEEL_URL,
+)
+
+
+@pytest.mark.parametrize(
+    "pkg_filename, pkg_url",
+    [(PYTHON_WHEEL_FILENAME, PYTHON_WHEEL_URL), (PYTHON_EGG_FILENAME, PYTHON_EGG_URL)],
+)
+def test_synchronous_package_upload(
+    delete_orphans_pre, download_python_file, gen_user, python_bindings, pkg_filename, pkg_url
+):
+    """
+    Test synchronously uploading a Python package with labels.
+    """
+    python_file = download_python_file(pkg_filename, pkg_url)
+
+    # Upload a unit with labels
+    with gen_user(model_roles=["python.python_package_uploader"]):
+        labels = {"key_1": "value_1"}
+        content_body = {"file": python_file, "pulp_labels": labels}
+        package = python_bindings.ContentPackagesApi.upload(**content_body)
+        assert package.pulp_labels == labels
+        assert package.name == "shelf-reader"
+        assert package.filename == pkg_filename
+
+    # Check that uploading the same unit again with different (or same) labels has no effect
+    with gen_user(model_roles=["python.python_package_uploader"]):
+        labels_2 = {"key_2": "value_2"}
+        content_body_2 = {"file": python_file, "pulp_labels": labels_2}
+        duplicate_package = python_bindings.ContentPackagesApi.upload(**content_body_2)
+        assert duplicate_package.pulp_href == package.pulp_href
+        assert duplicate_package.pulp_labels == package.pulp_labels
+        assert duplicate_package.pulp_labels != labels_2
+
+    # Check that the upload fails if the user does not have the required permissions
+    with gen_user(model_roles=[]):
+        with pytest.raises(python_bindings.ApiException) as ctx:
+            python_bindings.ContentPackagesApi.upload(**content_body)
+        assert ctx.value.status == 403

pyproject.toml

@@ -26,7 +26,7 @@ classifiers=[
 ]
 requires-python = ">=3.11"
 dependencies = [
-  "pulpcore>=3.49.0,<3.100",
+  "pulpcore>=3.81.0,<3.100",
   "pkginfo>=1.12.0,<1.13.0",
   "bandersnatch>=6.3.0,<6.6", # 6.6 has breaking changes
   "pypi-simple>=1.5.0,<2.0",