Migrate to minio

We preferred to choose `Minio` instead of `Localstack` due to the fact
that `localstack` does not support saving files after restarting the docker
container for the free version.
localstack/localstack#6281 (comment)

compose.yaml

@@ -1,45 +1,58 @@
 name: "saritasa-s3-tools"
 
+volumes:
+  minio_data:
+
 services:
   # ###########################################################################
-  # Localstack - solution for many aws services in one container
-  # Used for s3
-  # Docs: https://docs.localstack.cloud/overview/
+  # MinIO Storage
+  # minio:9001 - Server
+  # minio:9000 - UI
+
+  # We preferred to choose `Minio` instead of `Localstack` due to the fact
+  # that `localstack` does not support saving files after restarting the docker
+  # container for the free version.
+  # https://github.com/localstack/localstack/issues/6281#issuecomment-1169731265
   # ###########################################################################
-  localstack-services:
-    image: localstack/localstack:3.7.0
-    hostname: localhost.localstack.cloud
-    ports:
-      - "4566:4566"
-      - '9000:9000'
-      - '9001:9001'
+  minio:
+    image: minio/minio:latest
+    command: server --address 0.0.0.0:9001 --console-address 0.0.0.0:9000 /data
     environment:
-      # https://docs.localstack.cloud/references/configuration/
-      - SERVICES=s3
-      - DEBUG=1
-      - DOCKER_HOST=unix:///var/run/docker.sock
-      - AWS_ACCESS_KEY_ID=root
-      - AWS_SECRET_ACCESS_KEY=rootroot
-      - AWS_DEFAULT_REGION=us-west-1
+      - MINIO_ROOT_USER=root
+      - MINIO_ROOT_PASSWORD=rootroot
+      # Set `MINIO_DOMAIN` to enable virtual-hosted-style for minio.
+      # https://min.io/docs/minio/linux/administration/object-management.html#id1
+      - MINIO_DOMAIN=s3.minio.localhost
+    ports:
+      - "9001:9001"
+      - "9000:9000"
     volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./localstack_volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"
+      - minio_data:/data
+    networks:
+      default:
+        aliases:
+          # Set alias that suitable for virtual-hosted-style URL structure:
+          # `https://<bucket>.s3.<region>.amazonaws.com/<key>
+          # We need apply this style for local S3 storage because path-style
+          # URLs will be discontinued in the future, and we want to make
+          # consistent URLs for uploading and downloading files.
+          # https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
+          - ${COMPOSE_PROJECT_NAME}-files.s3.minio.localhost
 
-  localstack-services-s3:
-    image: localstack/localstack:3.7.0
+  # ###########################################################################
+  # Service for creating bucket in minio service
+  # ###########################################################################
+  minio-create-bucket:
+    image: minio/mc:latest
     depends_on:
-      - localstack-services
-    environment:
-      - AWS_DEFAULT_REGION=us-west-1
-      - AWS_ENDPOINT_URL=http://localhost.localstack.cloud:4566
+      - minio
     entrypoint: >
       /bin/sh -c "
-      awslocal s3api create-bucket --bucket ${COMPOSE_PROJECT_NAME}-files --region us-west-1 --create-bucket-configuration LocationConstraint=us-west-1;
+      /usr/bin/mc alias set s3minio http://minio:9001 root rootroot;
+      /usr/bin/mc mb s3minio/${COMPOSE_PROJECT_NAME}-files;
+      /usr/bin/mc anonymous set public s3minio/${COMPOSE_PROJECT_NAME}-files;
       exit 0;
       "
-    volumes:
-      - "${LOCALSTACK_VOLUME_DIR:-./localstack_volume}:/var/lib/localstack"
-      - "/var/run/docker.sock:/var/run/docker.sock"
 
   postgres:
     image: postgres:16

example/settings.py

@@ -118,7 +118,7 @@
 AWS_S3_ACCESS_KEY_ID = "root"
 AWS_S3_SECRET_ACCESS_KEY = "rootroot"  # noqa: S105
 AWS_S3_REGION_NAME = "us-west-1"
-AWS_S3_ENDPOINT_URL = "https://s3.localhost.localstack.cloud:4566"
+AWS_S3_ENDPOINT_URL = "http://s3.minio.localhost:9001"
 AWS_STORAGE_BUCKET_NAME = "saritasa-s3-tools-files"
 AWS_QUERYSTRING_AUTH = True
 AWS_S3_FILE_OVERWRITE = False

pyproject.toml

@@ -324,7 +324,7 @@ norecursedirs = [
 # set Django settings
 DJANGO_SETTINGS_MODULE = "example.settings"
 # Configuration for s3
-s3_endpoint_url = "https://localhost.localstack.cloud:4566"
+s3_endpoint_url = "http://s3.minio.localhost:9001"
 s3_region="us-west-1"
 s3_access_key="root"
 s3_secret_key="rootroot"

saritasa_s3_tools/testing/shortcuts.py

@@ -33,6 +33,7 @@ def upload_file(
 def upload_file_and_verify(
     filepath: str,
     s3_params: client.S3UploadParams,
+    is_minio: bool = False,
 ) -> tuple[str, str]:
     """Upload and verify that file is uploaded."""
     upload_response = upload_file(
@@ -43,8 +44,12 @@ def upload_file_and_verify(
     parsed_response = xml.etree.ElementTree.fromstring(  # noqa: S314
         upload_response.content.decode(),
     )
-    file_key = parsed_response[2].text
-    file_url = parsed_response[0].text
+    if is_minio:
+        file_key = parsed_response[1].text
+        file_url = parsed_response[3].text
+    else:
+        file_key = parsed_response[2].text
+        file_url = parsed_response[0].text
     assert file_url, upload_response.content  # noqa: S101
     assert file_key, upload_response.content  # noqa: S101
     return file_url, file_key

tasks.py

@@ -29,8 +29,8 @@
             project_name="saritasa-s3-tools",
             docker=saritasa_invocations.DockerSettings(
                 main_containers=(
-                    "localstack-services",
-                    "localstack-services-s3",
+                    "minio",
+                    "minio-create-bucket",
                     "postgres",
                 ),
             ),

tests/test_async_client.py

@@ -22,6 +22,7 @@ async def test_upload(
     _, file_key = saritasa_s3_tools.testing.upload_file_and_verify(
         filepath=__file__,
         s3_params=s3_params,
+        is_minio=True,
     )
     meta_data = await async_s3_client.async_get_file_metadata(
         key=file_key,

tests/test_client.py

@@ -23,6 +23,7 @@ def test_upload(s3_client: saritasa_s3_tools.S3Client) -> None:
     _, file_key = saritasa_s3_tools.testing.upload_file_and_verify(
         filepath=__file__,
         s3_params=s3_params,
+        is_minio=True,
     )
     meta_data = s3_client.get_file_metadata(key=file_key)
     assert meta_data["Metadata"]["config-name"] == "files"
@@ -109,7 +110,7 @@ def test_upload_expiration(s3_client: saritasa_s3_tools.S3Client) -> None:
         response.content.decode(),
     )[1].text
     assert (
-        error == "Invalid according to Policy: Policy expired."
+        error == "Access Denied. (Invalid according to Policy: Policy expired)"
     ), response.content
 
 

tests/test_django/test_file_upload.py

@@ -41,6 +41,7 @@ def test_file_upload(
     file_url, file_key = saritasa_s3_tools.testing.upload_file_and_verify(
         filepath=__file__,
         s3_params=saritasa_s3_tools.client.S3UploadParams(**response.data),
+        is_minio=True,
     )
     value_to_send: str
     match value:
@@ -80,6 +81,7 @@ def test_file_upload_invalid_config_used(
     file_url, file_key = saritasa_s3_tools.testing.upload_file_and_verify(
         filepath=__file__,
         s3_params=saritasa_s3_tools.client.S3UploadParams(**response.data),
+        is_minio=True,
     )
     response = api_client.post(
         path=reverse_lazy("model-api-list"),

tests/test_factory.py

@@ -16,7 +16,7 @@ def test_s3_file_factory(
             access_key="root",
             secret_key="rootroot",
         ),
-        s3_endpoint_url_getter=lambda: "https://localhost.localstack.cloud:4566",
+        s3_endpoint_url_getter=lambda: "http://s3.minio.localhost:9001",
     ).evaluate(object(), None, None)
     (
         s3_client.is_file_in_bucket(
@@ -40,7 +40,7 @@ def test_s3_image_factory(
             access_key="root",
             secret_key="rootroot",
         ),
-        s3_endpoint_url_getter=lambda: "https://localhost.localstack.cloud:4566",
+        s3_endpoint_url_getter=lambda: "http://s3.minio.localhost:9001",
     ).evaluate(object(), None, None)
     assert s3_client.is_file_in_bucket(
         file_key,