Merge pull request #68 from pvarki/apparent_race_condition

Deps update, improved logging

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.0
+current_version = 1.4.1
 commit = False
 tag = False
 

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ default_language_version:
     python: python3
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: no-commit-to-branch
     -   id: check-executables-have-shebangs
@@ -22,29 +22,30 @@ repos:
         args:
         - --autofix
 -   repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.1.1
     hooks:
     -   id: black
         language_version: python3
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.5.1
+    rev: v1.8.0
     hooks:
     -   id: mypy
         language: system
+        args: ["--strict"]
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.17.5
+    rev: v3.0.3
     hooks:
     -   id: pylint
         language: system
 -   repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.5.0
+    rev: v1.5.4
     hooks:
     -   id: forbid-crlf
     -   id: remove-crlf
     -   id: forbid-tabs
     -   id: remove-tabs
 -   repo: https://github.com/PyCQA/bandit
-    rev: 1.7.5
+    rev: 1.7.7
     hooks:
     - id: bandit
       args: ["--skip=B101"]
@@ -59,3 +60,13 @@ repos:
         language: system
         exclude: "poetry\\.lock|tests/data/.*"
         # args: ['--baseline', '.secrets.baseline']
+-   repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.9.0.6
+    hooks:
+    -   id: shellcheck
+        args: ["--external-sources"]
+-   repo: https://github.com/python-poetry/poetry
+    rev: '2.0.1'
+    hooks:
+    -   id: poetry-check
+    -   id: poetry-lock

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1.1.7-experimental
 ARG TEMURIN_VERSION="17"
-ARG TAKSERVER_IMAGE="pvarki/takserver:4.10-RELEASE-12-20240302"
+ARG TAKSERVER_IMAGE="pvarki/takserver:5.3-RELEASE-24"
 
 # The local reference tak_server is used in future stages
 FROM ${TAKSERVER_IMAGE} as tak_server
@@ -10,7 +10,7 @@ FROM ${TAKSERVER_IMAGE} as tak_server
 #############################################
 FROM advian/tox-base:debian-bookworm as tox
 ARG PYTHON_VERSIONS="3.11 3.10 3.9 3.11"
-ARG POETRY_VERSION="1.5.1"
+ARG POETRY_VERSION="2.0.1"
 RUN export RESOLVED_VERSIONS=`pyenv_resolve $PYTHON_VERSIONS` \
     && echo RESOLVED_VERSIONS=$RESOLVED_VERSIONS \
     && for pyver in $RESOLVED_VERSIONS; do pyenv install -s $pyver; done \
@@ -41,7 +41,7 @@ ENV \
   PIP_DISABLE_PIP_VERSION_CHECK=on \
   PIP_DEFAULT_TIMEOUT=100 \
   # poetry:
-  POETRY_VERSION=1.5.1
+  POETRY_VERSION=2.0.1
 
 
 RUN apt-get update && apt-get install -y \
@@ -75,6 +75,7 @@ WORKDIR /pysetup
 COPY ./poetry.lock ./pyproject.toml /pysetup/
 # Install basic requirements (utilizing an internal docker wheelhouse if available)
 RUN --mount=type=ssh pip3 install wheel virtualenv \
+    && poetry self add poetry-plugin-export \
     && poetry export -f requirements.txt --without-hashes -o /tmp/requirements.txt \
     && pip3 wheel --wheel-dir=/tmp/wheelhouse -r /tmp/requirements.txt \
     && virtualenv /.venv && source /.venv/bin/activate && echo 'source /.venv/bin/activate' >>/root/.profile \
@@ -145,7 +146,8 @@ ENTRYPOINT ["/usr/bin/tini", "--", "/docker-entrypoint.sh"]
 #####################################
 FROM builder_base as devel_build
 # Install deps
-WORKDIR /pysetup
+COPY . /app
+WORKDIR /app
 RUN --mount=type=ssh source /.venv/bin/activate \
     && poetry install --no-interaction --no-ansi \
     && true

docker/entrypoint-test.sh

@@ -1,5 +1,6 @@
 #!/bin/bash -l
 set -e
+# shellcheck disable=SC1091
 source /.venv/bin/activate
 if [ "$#" -eq 0 ]; then
   # Kill cache, pytest complains about it if running local and docker tests in mapped volume

docker/entrypoint.sh

@@ -1,7 +1,6 @@
 #!/bin/bash -l
+# shellcheck disable=SC1091
 . /container-init.sh
-TR=/opt/tak
-CR=${TR}/certs
 
 if [ "$#" -eq 0 ]; then
   exec gunicorn "takrmapi.app:get_app()" --bind 0.0.0.0:8003 --forwarded-allow-ips='*' -w 4 -k uvicorn.workers.UvicornWorker

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "takrmapi"
-version = "1.4.0"
+version = "1.4.1"
 description = "RASENMAEHER integration API for TAK server"
 authors = ["Eero af Heurlin <[email protected]>", "Ari Karhunen <[email protected]>"]
 homepage = "https://github.com/pvarki/python-tak-rmapi"
@@ -61,23 +61,23 @@ branch = true
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
 libadvian = "^1.4"
-libpvarki = { git="https://github.com/pvarki/python-libpvarki.git", tag="1.9.0" }
+libpvarki = { git="https://github.com/pvarki/python-libpvarki.git", tag="1.9.1" }
 fastapi = ">=0.89,<1.0,!=0.99.0"  # caret behaviour on 0.x is to lock to 0.x.*
 jinja2 = "^3.1"
-uvicorn = {version = "^0.20", extras = ["standard"]}
-gunicorn = "^20.1"
+uvicorn = {version = ">=0.30,<1.0", extras = ["standard"]}
+gunicorn = "^23.0"
 filelock = "^3.12"
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.1"
-coverage = "^6.3"
-pytest-cov = "^3.0"
-pylint = "^2.13"
-black = "^22.3"
+pytest = "^8.0"
+coverage = "^7.4"
+pytest-cov = "^4.1"
+pylint = "^3.0"
+black = "^24.1"
 bandit = "^1.7"
-mypy = "^1.5"
-pre-commit = "^2.17"
-pytest-asyncio = "^0.18"
+mypy = "^1.8"
+pre-commit = "^3.6"
+pytest-asyncio = ">=0.21,<1.0" # caret behaviour on 0.x is to lock to 0.x.*
 bump2version = "^1.0"
 detect-secrets = "^1.2"
 httpx = ">=0.23,<1.0" # caret behaviour on 0.x is to lock to 0.x.*

src/takrmapi/__init__.py

@@ -1,2 +1,3 @@
 """ RASENMAEHER integration API for TAK server """
-__version__ = "1.4.0"  # NOTE Use `bump2version --config-file patch` to bump versions correctly
+
+__version__ = "1.4.1"  # NOTE Use `bump2version --config-file patch` to bump versions correctly

src/takrmapi/api/__init__.py

@@ -1,4 +1,5 @@
 """Api endpoints"""
+
 from fastapi.routing import APIRouter
 
 from .usercrud import router as usercrud_router

src/takrmapi/api/admininfo.py

@@ -1,4 +1,5 @@
 """Endpoints for information for the admin"""
+
 import logging
 
 from fastapi import APIRouter, Depends

src/takrmapi/api/clientinfo.py

@@ -1,4 +1,5 @@
 """Endpoints for information for the end-user"""
+
 import logging
 import base64
 from typing import List, Dict
@@ -19,7 +20,7 @@ async def client_instruction_fragment(user: UserCRUDRequest) -> List[Dict[str, s
     """Return zip package containing client config and certificates"""
     localuser = tak_helpers.UserCRUD(user)
     tak_missionpkg = tak_helpers.MissionZip(localuser)
-    zip_files = await tak_missionpkg.create_missionpkg()
+    zip_files, tmp_folder = await tak_missionpkg.create_missionpkg()
     returnable: List[Dict[str, str]] = []
     for file in zip_files:
         with open(file, "rb") as filehandle:
@@ -32,4 +33,5 @@ async def client_instruction_fragment(user: UserCRUDRequest) -> List[Dict[str, s
                 "filename": f"{user.callsign}_{filename}",
             }
         )
+    await tak_missionpkg.helpers.remove_tmp_dir(str(tmp_folder))
     return returnable

src/takrmapi/api/description.py

@@ -1,4 +1,5 @@
 """Descriptions API"""
+
 from typing import Optional
 import logging
 

src/takrmapi/api/healthcheck.py

@@ -1,4 +1,5 @@
 """Endpoints for information for the admin"""
+
 from typing import Optional
 import logging
 import os

src/takrmapi/api/instructions.py

@@ -1,4 +1,5 @@
 """Instructions endpoints"""
+
 from typing import Dict
 import logging
 

src/takrmapi/api/usercrud.py

@@ -1,4 +1,5 @@
 """"User actions"""
+
 import logging
 from fastapi import APIRouter, Depends
 from libpvarki.middleware import MTLSHeader

src/takrmapi/app.py

@@ -1,4 +1,5 @@
 """"factory for the fastpi app"""
+
 from typing import AsyncGenerator
 import logging
 from contextlib import asynccontextmanager

src/takrmapi/config.py

@@ -1,4 +1,5 @@
 """Configurations with .env support"""
+
 from typing import Dict, Any, cast
 from pathlib import Path
 import json
@@ -40,9 +41,11 @@ def read_deployment_name() -> str:
     return str(load_manifest()["deployment"])
 
 
-cfg = Config()  # not supporting .env files anymore because https://github.com/encode/starlette/discussions/2446
+cfg = Config(
+    env_prefix="TI"
+)  # not supporting .env files anymore because https://github.com/encode/starlette/discussions/2446
 
-LOG_LEVEL: int = cfg("LOG_LEVEL", default=20, cast=int)
+LOG_LEVEL: int = cfg("LOG_LEVEL", default=10, cast=int)
 TEMPLATES_PATH: Path = cfg("TEMPLATES_PATH", cast=Path, default=Path(__file__).parent / "templates")
 
 TAK_CERTS_FOLDER: Path = cfg("TAK_CERTS_FOLDER", cast=Path, default=Path("/opt/tak/data/certs/files"))

src/takrmapi/console.py

@@ -1,4 +1,5 @@
 """CLI entrypoints for product integration api"""
+
 import asyncio
 import logging
 import json