Fixes truncate method and adds some tooling (#1)

* Fixes truncate method by rewriting 3 SQL statements into one so it can
work also in AUTOCOMMIT mode.
* Adds Makefile for easy tooling
* Adds a script that will copy the setup.cfg version to the module

Author: Red-HAP

Diff for: MANIFEST.in

@@ -0,0 +1 @@
+include src/asyncpg_lostream/__version__

Diff for: Makefile

@@ -0,0 +1,93 @@
+#  Copyright 2022 Red Hat, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# Makefile for standardized tasks.
+
+help:
+	@echo "Targets for asyncpg-lostream development:"
+	@echo "  start_db             : Start up the test database container (uses port 5432)"
+	@echo "  stop_db              : Stop the test database container (destroys the container volume also)"
+	@echo "  verify_db            : Verify that the database in the container is accepting connections"
+	@echo "  run_tests            : Execute 'verify_db', then execute pytest"
+	@echo "  clean                : Execute 'stop_db' then remove build output disk files"
+	@echo "  test                 : Execute 'clean', 'start_db', 'run_tests', 'stop_db'"
+	@echo "  sync_version         : Sync the module version file to the version in setup.cfg"
+	@echo "  build                : Execute 'clean', 'test', 'sync_version' then execute the build"
+	@echo "  verify_dist_dir      : Ensure that the './dist' dir exists"
+	@echo "  verify_dist_contents : Ensure that the './dist' dir is not empty"
+	@echo "  verify_artifacts     : Execute 'verify_dist_dir', 'verify_dist_contents'"
+	@echo "  twine_check          : Validate that the artifacts can be successfully uploaded to PyPI"
+	@echo
+
+all: clean
+	@make build
+
+verify_db:
+	@printf "Checking availability of test database: "
+	@while ! docker exec apl-postgres pg_isready -qd apl_postgres; do printf "."; sleep 1; done
+	@echo " PostgreSQL database is available."
+
+start_db:
+	@echo "Starting up test database..."
+	@docker-compose -f ./docker/docker-compose.yml up -d postgres
+
+stop_db:
+	@echo "Shutting down test database..."
+	@docker-compose -f ./docker/docker-compose.yml down -v
+
+clean: stop_db
+	@echo "Cleaning up any former builds..."
+	@rm -rf ./dist
+	@rm -rf ./src/asyncpg_lostream.egg-info/
+
+run_tests: verify_db
+	@echo "Running tests..."
+	@python -m pytest
+
+test: clean start_db run_tests
+	@make stop_db
+
+sync_version:
+	@echo "Syncing version from setup.cfg..."
+	@./scripts/sync_version.py
+
+build: clean test sync_version
+	@echo "Building targets..."
+	@python -m build
+
+verify_dist_dir:
+	@if [[ ! -d ./dist ]]; then echo "The 'dist' directory does not exist."; false; else true; fi
+
+verify_dist_contents:
+	@if [[ ! "$(shell ls -A ./dist)" ]]; then echo "The 'dist' directory is empty."; false; else true; fi
+
+verify_artifacts: verify_dist_dir
+	@make verify_dist_contents
+
+twine_check: verify_artifacts
+	@echo "Making sure targets will successfully upload."
+	@python -m twine check --strict ./dist/*
+
+
+
+# ==============================================================================
+# ==============================================================================
+# For the code owner(s) only!!
+deploy_testpypi: twine_check
+	@echo "Deploying to testpypi..."
+	@python -m twine upload --verbose --repository testpypi dist/*
+
+deploy_pypi: twine_check
+	@echo "Deploying to testpypi..."
+	@python -m twine upload --verbose --repository pypi dist/*

Diff for: README.md

@@ -95,16 +95,22 @@ async with PGLargeObject(async_connection, existing_lob_oid, "r") as pgl:
 
 ### Development
 
-Start a local PostgreSQL container using `docker-compose -f ./docker/docker-compose.yml up -d`
+`make`, `docker-compose` and `docker` are required for development.
 
-After making changes, create your unit tests in the `asyncpg-lostream/tests` directory.
+To list the make targets, use `make help`
+
+To start a local PostgreSQL 13 container: `make start_db`
 
-Test your changes with the command `python -m pytest`
+To shutdown the local PostgreSQL 13 container: `make stop_db`
 
-Shutdown the local PostgreSQL container `docker-compose -f ./docker/docker-compose.yml down -v`
+After making changes, create your unit tests in the `asyncpg-lostream/tests` directory.
+
+Test your changes with the command `make test`
 
 ## Packaging
 
-Build the package using `python -m build`. This will put built packages into the `dist/` directory.
+If you intend to make a releaase, change the version in the `setup.cfg` file. This value will be copied to the module's `__version__` file.
+
+Build the package using `make build`. This will run the tests and then build the artifacts. These will be put into the `dist/` directory.
 
 For instructions on upload to PyPI, see the [Packaging Python Projects Dcoumentation](https://packaging.python.org/en/latest/tutorials/packaging-projects/#uploading-the-distribution-archives)

Diff for: requirements.txt

@@ -9,3 +9,7 @@ pytest
 pytest-asyncio
 twine
 build
+wheel
+configparser
+packaging
+

Diff for: scripts/sync_version.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+#  Copyright 2022 Red Hat, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import configparser
+import pathlib
+
+from packaging import version
+
+MY_DIR = pathlib.Path(__file__).parent
+BASE_DIR = MY_DIR.parent
+SETUP_CFG = BASE_DIR / "setup.cfg"
+ASYNCPG_LOSTREAM = BASE_DIR / "src" / "asyncpg_lostream" / "__version__"
+
+
+setup_config = configparser.ConfigParser()
+setup_config.read(SETUP_CFG)
+if not ASYNCPG_LOSTREAM.exists():
+    mod_version = "0.0.0"
+else:
+    mod_version = open(ASYNCPG_LOSTREAM, "rt").read().strip()
+
+cfg_ver = version.Version(setup_config["metadata"]["version"])
+mod_ver = version.Version(mod_version)
+
+if cfg_ver != mod_ver:
+    print(f"Copying setup.cfg version {cfg_ver} to __version__ file")
+    with open(ASYNCPG_LOSTREAM, "wt") as ver_file:
+        ver_file.write(setup_config["metadata"]["version"])

Diff for: setup.cfg

@@ -1,6 +1,8 @@
 [metadata]
 name = asyncpg-lostream
-version = 0.1.1
+# Changing the version here will change the value in the module version
+# if you use the Makefile to build the artifacts.
+version = 0.1.2b1
 author = Red Hat, Inc.
 author_email = [email protected]
 maintainer = HAP
@@ -40,6 +42,7 @@ package_dir =
 python_requires = >=3.9
 install_requires =
     sqlalchemy[asyncio] >= 1.4
+    asyncpg
 
 [options.packages.find]
 where = src

Diff for: src/asyncpg_lostream/__init__.py

@@ -0,0 +1,5 @@
+import pathlib
+
+MY_DIR = pathlib.Path(__file__).parent
+
+VERSION = open(MY_DIR / "__version__", "rt").read().strip()

Diff for: src/asyncpg_lostream/__version__

@@ -0,0 +1 @@
+0.1.2b1

Diff for: src/asyncpg_lostream/lostream.py

@@ -15,7 +15,7 @@
 from __future__ import annotations
 
 import logging
-from typing import AsyncIterator, List, Optional, Tuple, Union
+from typing import AsyncIterator, List, Optional, Tuple
 
 from sqlalchemy import column, func, select, text
 from sqlalchemy.dialects.postgresql import ARRAY, OID, array
@@ -35,6 +35,7 @@
 
 class PGLargeObjectError(Exception):
     """Base PGLargeObject Exception"""
+
     pass
 
 
@@ -100,9 +101,7 @@ def __init__(
     ):
         self.session = session
         if chunk_size <= 0:
-            raise ValueError(
-                "chunk_size must be an integer greater than zero."
-            )
+            raise ValueError("chunk_size must be an integer greater than zero.")
         self.chunk_size = chunk_size
         self.oid = oid
         self.closed = False
@@ -130,9 +129,7 @@ async def __anext__(self) -> bytes:
     def resolve_mode(mode: str) -> Tuple[int, bool]:
         """Translates string mode to the proper binary setting."""
         if mode not in VALID_MODES:
-            raise ValueError(
-                f"Mode {mode} must be one of {sorted(VALID_MODES)}"
-            )
+            raise ValueError(f"Mode {mode} must be one of {sorted(VALID_MODES)}")
 
         append = mode.startswith("a")
         imode = 0
@@ -147,23 +144,16 @@ async def create_large_object(session: AsyncSession) -> int:
         sql = select(func.lo_create(0).label("loid"))
         oid = await session.scalar(sql)
         if oid == 0:
-            raise PGLargeObjectNotCreated(
-                "Requested large object was not created."
-            )
+            raise PGLargeObjectNotCreated("Requested large object was not created.")
+        LOG.debug(f"Created large object with oid = {oid}")
         return oid
 
     @staticmethod
-    async def verify_large_object(
-        session: AsyncSession, oid: int
-    ) -> Tuple[bool, int]:
+    async def verify_large_object(session: AsyncSession, oid: int) -> Tuple[bool, int]:
         """Verify that a large object exists. Returns oid and size."""
         sel_cols = [
             column("oid"),
-            sum_(
-                func.length(
-                    coalesce(text("data"), func.convert_to("", "utf-8"))
-                )
-            ).label("data_length"),
+            sum_(func.length(coalesce(text("data"), func.convert_to("", "utf-8")))).label("data_length"),
         ]
         sql = (
             select(sel_cols)
@@ -182,15 +172,14 @@ async def verify_large_object(
         return rec
 
     @staticmethod
-    async def delete_large_object(
-        session: AsyncSession, oids: List[int]
-    ) -> None:
+    async def delete_large_object(session: AsyncSession, oids: List[int]) -> None:
         """Deletes large objects. This will deallocate the large object oids specified."""
         if oids and all(o > 0 for o in oids):
             sql = select(func.lo_unlink(text("oid"))).select_from(
                 func.unnest(array(oids, type_=ARRAY(OID))).alias("oid")
             )
             await session.execute(sql)
+            LOG.debug(f"Deleted large objects with these OIDs: {oids}")
 
     def closed_check(self):
         """Verify that an object interface is closed."""
@@ -202,9 +191,7 @@ async def read(self, chunk_size: Optional[int] = None) -> bytes:
         self.closed_check()
 
         if self.imode == INV_WRITE:
-            raise PGLargeObjectUnsupportedOp(
-                "Large Object class instance is set for write only"
-            )
+            raise PGLargeObjectUnsupportedOp("Large Object class instance is set for write only")
 
         _chunk_size = chunk_size or self.chunk_size
 
@@ -222,7 +209,7 @@ def _get_buffer_chunk(self, buffer: bytes, chunk_size: int) -> bytes:
         start = 0
         chunk = None
         while True:
-            chunk = buffer[start:(start + chunk_size)]
+            chunk = buffer[start : (start + chunk_size)]  # noqa: E203
             if not chunk:
                 break
             start += chunk_size
@@ -233,9 +220,7 @@ async def write(self, buff: bytes, chunk_size: Optional[int] = None) -> int:
         self.closed_check()
 
         if self.imode == INV_READ:
-            raise PGLargeObjectUnsupportedOp(
-                "Large Object class instance is set for read only"
-            )
+            raise PGLargeObjectUnsupportedOp("Large Object class instance is set for read only")
 
         if not buff:
             return 0
@@ -260,13 +245,13 @@ async def truncate(self) -> None:
         if self.imode == INV_READ:
             raise PGLargeObjectUnsupportedOp("not writeable")
 
-        open_sql = select(func.lo_open(self.oid, self.imode))
-        fd = await self.session.scalar(open_sql)
-
-        trunc_sql = select(func.lo_truncate(fd, self.pos))
-        await self.session.execute(trunc_sql)
-
-        close_sql = select(func.lo_close(fd))
+        open_cte = select(func.lo_open(self.oid, self.imode).label("descriptor")).cte("fd")
+        trunc_cte = (
+            select(func.lo_truncate(open_cte.c.descriptor, self.pos).label("res"), open_cte.c.descriptor)
+            .select_from(open_cte)
+            .cte("trnc")
+        )
+        close_sql = select(func.lo_close(trunc_cte.c.descriptor)).select_from(trunc_cte)
         await self.session.execute(close_sql)
 
         self.length = self.pos
@@ -288,9 +273,7 @@ async def open(self) -> None:
             self.length = length
         else:
             if not self.imode & INV_WRITE:
-                raise PGLargeObjectNotFound(
-                    f"Large object with oid {self.oid} does not exist."
-                )
+                raise PGLargeObjectNotFound(f"Large object with oid {self.oid} does not exist.")
             else:
                 self.oid = await self.create_large_object(self.session)
                 self.length = self.pos = 0

Diff for: tests/integration/conftest.py

@@ -23,8 +23,6 @@
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
 
-# from .utils.db import create_database, drop_database
-
 
 class Settings:
     database_url = "postgresql+asyncpg://postgres:secret@localhost:5432/postgres"