Skip to content

Commit 64567a3

Browse files
hanstromperthanstrompert
authored
fixes, updates, and upgrades (#280)
* add environment variable to enable echoing of sql statements * fix database locked error by always explicitly beginning transaction * improve resources in use/free detection for both reserve and modify * check for unuported parameters on modify and return message payload error * refactor setup.py and setup.cfg into a pyproject.toml by using build-system independent format (PEP 517) * improve and extend unit tests - add session wide thread pool executor and reserve failed fixtures - add new functional test suite - fix tests after strict parameter checking on modify requests - skip job reserve test that should be rewritten into functional one * refactor Dockerfile to perform proper staged build * improve locating backend module on sys path - first look for backend module directly on sys path - else look for backend module in installed supa package (supa.nrm.backends) on sys path * update github workflows - install of code does not need to be editable - use new way of compiling proto files - split unit tests with mypy in two separate runs * update container build - switch from debian to alpine as build and runtime base image - upgrade to python 3.14 * update dependencies - bump apscheduler from 3.11.0 to 3.11.1 - bump certifi from 2025.10.5 to 2025.11.12 - bump charset-normalizer from 3.4.3 to 3.4.4 - bump cheroot from 11.0.0 to 11.1.2 - bump click from 8.3.0 to 8.3.1 - bump cryptography from 46.0.2 to 46.0.3 - bump idna from 3.10 to 3.11 - bump invoke from 2.2.0 to 2.2.1 - bump mypy-protobuf from 3.6.0 to 3.7.0 - bump protobuf from 6.32.1 to 6.33.1 - bump pydantic from 2.12.0 to 2.12.4 - bump pydantic-core from 2.41.1 to 2.41.5 - bump pydantic-settings from 2.11.0 to 2.12.0 - bump pynacl from 1.6.0 to 1.6.1 - bump python-dotenv from 1.1.1 to 1.2.1 - bump sqlalchemy from 2.0.43 to 2.0.44 - bump structlog from 25.4.0 to 25.5.0 - bump types-protobuf from 6.32.1.20250918 to 6.32.1.20251105 - remove grpcio-tools
1 parent 9cd0e5a commit 64567a3

File tree

24 files changed

+1071
-419
lines changed

24 files changed

+1071
-419
lines changed

‎.github/workflows/run-linting-tests.yml‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,18 @@ jobs:
2525
- name: Install dependencies
2626
run: |
2727
python -m pip install --upgrade pip
28-
pip install -e '.[dev]'
29-
python setup.py gen_code
28+
pip install '.[dev]'
29+
python src/supa/buildtools/backend.py # compile proto files
3030
- name: Check formatting
3131
run: |
32-
isort -c .
33-
black --check .
32+
isort -c src/supa tests
33+
black --check src/supa tests
3434
- name: Lint with flake8
3535
run: |
3636
# stop the build if there are Python syntax errors or undefined names
37-
flake8 .
37+
flake8 src/supa tests
3838
- name: Check with mypy
3939
run: |
40+
# do not run mypy on both directories at the same time
4041
mypy src/supa
42+
mypy tests

‎.github/workflows/run-unit-tests.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ jobs:
1818
apt update
1919
apt install -y curl git
2020
python -m pip install --upgrade pip
21-
pip install -e '.[dev]'
22-
python setup.py gen_code
21+
pip install '.[dev]'
22+
python src/supa/buildtools/backend.py # compile proto files
2323
- name: Run Unit tests
2424
run: pytest --cov-report term-missing --cov-report=xml --cov=src tests
2525
- name: "Upload coverage to Codecov"

‎Dockerfile‎

Lines changed: 27 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,32 @@
11
# syntax=docker/dockerfile:1
22
#
3-
FROM python:3.12-slim-bookworm AS base
4-
RUN set -ex; apt-get update && apt-get upgrade -y && rm -rf /var/lib/apt/lists/*
5-
ENV BASEDIR=/usr/local/src/supa
6-
WORKDIR $BASEDIR
7-
COPY requirements.txt requirements.txt
8-
RUN pip install -U pip wheel
9-
RUN pip install -r requirements.txt
3+
# Build stage
4+
FROM python:3.14-alpine AS build
5+
#
6+
ENV PIP_ROOT_USER_ACTION=ignore
7+
#
8+
WORKDIR /app
9+
RUN set -ex; apk update && apk upgrade
10+
RUN pip install --upgrade pip build --no-cache-dir
11+
COPY pyproject.toml LICENSE.txt README.rst supa.env .
12+
COPY src src
13+
RUN python -m build --wheel --outdir dist
1014

11-
FROM base AS final
12-
COPY . .
13-
RUN python setup.py gen_code
14-
RUN python setup.py install
15+
# Final stage
16+
FROM python:3.14-alpine
17+
#
18+
ENV PIP_ROOT_USER_ACTION=ignore
19+
ENV DATABASE_DIR=/usr/local/var/db
20+
#
21+
RUN set -ex; apk update && apk upgrade --no-cache
22+
RUN pip install --upgrade pip --no-cache-dir
23+
COPY --from=build /app/dist/*.whl /tmp/
24+
RUN pip install /tmp/*.whl --no-cache-dir && rm /tmp/*.whl
25+
RUN addgroup -g 1000 supa && adduser -D -u 1000 -G supa supa
26+
RUN mkdir --parents $DATABASE_DIR && chown supa:supa $DATABASE_DIR
27+
USER supa
28+
#
1529
EXPOSE 8080/tcp 50051/tcp
16-
ENV PYTHONPATH=/usr/local/etc/supa:$BASEDIR/src/supa/nrm/backends
30+
ENV PYTHONPATH=/usr/local/etc/supa
31+
ENV TZ=UTC
1732
CMD ["supa", "serve"]

‎docs/development.rst‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -117,17 +117,17 @@ Some rules
117117
Importing new protobuf/gRPC definitions
118118
---------------------------------------
119119

120-
When new NSI protobuf/gRPC definitions are imported into the ``protos`` directory
120+
When new NSI protobuf/gRPC definitions are imported into the ``src/supa/protos`` directory
121121
one should (re)generated the corresponding Python code for it::
122122

123-
% python setup.py clean gen_code
123+
% python src/supa/buildtools/backend.py
124124

125125
.. note::
126126

127-
Cleaning the previously generated code is a good thing thing.
127+
Cleaning the previously generated code is a good thing.
128128
We want to ensure that we don't accidentally depend on no longer used protobuf/gRPC definitions.
129-
Hence always run the ``gen_code`` in conjunction with
130-
and prepended by the ``clean`` command.
129+
Hence always remove the ``src/supa/protos`` directory
130+
before generating code from new or updated protos'
131131

132132

133133
PyCharm

‎docs/installation.rst‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ There are several methods to install and run the application:
1818

1919
.. note::
2020

21-
SuPA has been tested with Python 3.11 and 3.12.
21+
SuPA has been tested with Python version 3.11 upto and including 3.14.
2222
Compatibility with earlier Python versions will brake in the future or is already broken.
2323
Compatibility with later Python versions is untested.
2424

@@ -34,11 +34,11 @@ SuPA
3434

3535
_`Clone the SuPA source code repository from GitHub`,
3636
and checkout the desired branch or version,
37-
for example version 0.2.0::
37+
for example version 0.3.4::
3838

3939
git clone https://github.com/workfloworchestrator/supa.git
4040
cd supa
41-
git checkout 0.2.0
41+
git checkout 0.3.4
4242

4343
There are multiple ways to add an virtual Python environment, and you can choose
4444
whatever suits you best. This example uses the standard ``venv`` module and assumes

‎pyproject.toml‎

Lines changed: 212 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,140 @@ requires = [
77
"setuptools>=65.5.0",
88
"wheel",
99
"grpcio-tools==1.75.1",
10-
"mypy-protobuf",
11-
"bandit==1.7.10",
10+
"mypy_protobuf",
11+
]
12+
build-backend = "buildtools.backend"
13+
backend-path = ["src/supa"]
14+
15+
16+
[project]
17+
name = "supa"
18+
version = "0.3.5.dev1"
19+
dependencies = [
20+
"grpcio == 1.75.1",
21+
"mypy-protobuf ~= 3.0",
22+
"structlog ~= 25.0",
23+
"colorama ~= 0.0",
24+
"click ~= 8.0",
25+
"pydantic-settings ~= 2.0",
26+
"python-statemachine ~= 2.0",
27+
"sqlalchemy ~= 2.0",
28+
"apscheduler ~= 3.0",
29+
"tabulate ~= 0.0",
30+
"more-itertools ~= 10.0",
31+
"protobuf ~= 6.0",
32+
"python-nso-client ~= 0.0",
33+
"requests ~= 2.0",
34+
"lxml ~= 6.0",
35+
"lxml-stubs ~= 0.0",
36+
"cherrypy ~= 18.0",
37+
"htmlgen ~= 2.0",
38+
"paramiko >=3.0",
39+
"pytz ~= 2025.0",
40+
]
41+
requires-python = ">=3.11"
42+
authors = [
43+
{name = "Guido Kollerie", email = "[email protected]"},
44+
{name = "Hans Trompert", email = "[email protected]"},
45+
]
46+
maintainers = [
47+
{name = "Hans Trompert"},
48+
]
49+
description = "The SURF ultimate Provider Agent (SuPA)"
50+
readme = "README.rst"
51+
keywords = [
52+
"NSI",
53+
"Python",
54+
"uPA",
55+
"NSA",
56+
"gRPC",
57+
"protobuf",
58+
]
59+
classifiers = [
60+
"Development Status :: 4 - Beta",
61+
"Intended Audience :: Telecommunications Industry",
62+
"Operating System :: OS Independent",
63+
"Programming Language :: Python :: 3.11",
64+
"Programming Language :: Python :: 3.12",
65+
"Programming Language :: Python :: 3.13",
66+
"Topic :: System :: Networking",
67+
]
68+
license = "Apache-2.0"
69+
license-files = ["LICENSE.txt"]
70+
71+
[project.urls]
72+
#Homepage = "https://example.com"
73+
Documentation = "https://workfloworchestrator.org/SuPA"
74+
Repository = "https://github.com/workfloworchestrator/SuPA.git"
75+
Issues = "https://github.com/workfloworchestrator/SuPA/issues"
76+
#Changelog = "https://github.com/me/spam/blob/master/CHANGELOG.md"
77+
78+
[project.optional-dependencies]
79+
dev = [
80+
"bandit",
81+
"black",
82+
"flake8",
83+
"flake8-bandit", # security checks
84+
"flake8-bugbear", # assorted opinionated checks
85+
"flake8-builtins", # check for name collision with builtins
86+
"flake8-comprehensions",
87+
"flake8-docstrings",
88+
"flake8-implicit-str-concat",
89+
"flake8-print",
90+
"flake8-rst", # Allows run flake8 on code snippets in docstrings or rst files
91+
"flake8-string-format",
92+
"grpcio-tools == 1.75.1", # version number should match that of `grpcio-tools` in `build-system` above
93+
"grpc-stubs >= 1.53.0",
94+
"pytest-grpc",
95+
"isort",
96+
"mypy",
97+
"protobuf",
98+
"types-pytz",
99+
"types-tabulate",
100+
"pip-tools",
101+
"pre-commit",
102+
"pytest",
103+
"pytest-cov",
104+
"pytest-xdist",
105+
"types-requests",
106+
"types-setuptools",
107+
"types-paramiko >=3.0.0",
108+
]
109+
doc = [
110+
"sphinx",
111+
"sphinx-rtd-theme",
112+
"sphinx-click",
113+
]
114+
115+
[project.scripts]
116+
supa = "supa.main:cli"
117+
118+
[tool.setuptools]
119+
package-dir = {"" = "src"}
120+
121+
[tool.setuptools.package-data]
122+
"supa.protos" = ["*.proto"]
123+
124+
[tool.setuptools.packages.find]
125+
where = ["src"]
126+
127+
[tool.setuptools.data-files]
128+
"etc/supa" = ["supa.env", "src/supa/nrm/backends/*.env"]
129+
130+
[tool.pytest]
131+
minversion = "9.0"
132+
testpaths = [
133+
"tests",
134+
]
135+
136+
[tool.coverage.run]
137+
branch = true
138+
source = [
139+
"src/supa",
140+
]
141+
omit = [
142+
"src/supa/grpc_nsi/*",
143+
"tests/*",
12144
]
13145

14146
[tool.isort]
@@ -39,3 +171,81 @@ exclude = '''
39171
)/
40172
)
41173
'''
174+
175+
[tool.mypy]
176+
ignore_missing_imports = true
177+
disallow_untyped_calls = false
178+
disallow_untyped_defs = true
179+
disallow_incomplete_defs= true
180+
disallow_untyped_decorators = true
181+
no_implicit_optional = true
182+
warn_redundant_casts = false
183+
warn_unused_ignores = true
184+
warn_return_any = true
185+
warn_unreachable = true
186+
strict_equality = true
187+
show_error_codes = true
188+
show_column_numbers = true
189+
exclude = [
190+
"src/supa/nrm/backends/nso_service_model",
191+
]
192+
193+
[[tool.mypy.overrides]]
194+
module = [
195+
"supa.grpc_nsi.*",
196+
]
197+
disallow_untyped_defs = false
198+
disallow_incomplete_defs = false
199+
disallow_untyped_decorators = false
200+
201+
[tool.ruff]
202+
lint.exclude = [
203+
".git",
204+
".cache",
205+
".eggs",
206+
"__pycache__",
207+
"venv",
208+
".venv",
209+
]
210+
lint.ignore = [
211+
"D100",
212+
"D104",
213+
"T201",
214+
]
215+
line-length = 120
216+
lint.select = [
217+
"ANN",
218+
"ARG",
219+
"B",
220+
"C",
221+
"D",
222+
"E",
223+
"F",
224+
"I",
225+
"N",
226+
"PGH",
227+
"PTH",
228+
"RET",
229+
"RUF",
230+
"S",
231+
"T",
232+
"W",
233+
"Q",
234+
]
235+
target-version = "py311"
236+
237+
[tool.ruff.lint.flake8-tidy-imports]
238+
ban-relative-imports = "all"
239+
240+
[tool.ruff.lint.per-file-ignores]
241+
"tests/*" = ["S101"]
242+
243+
[tool.ruff.lint.pydocstyle]
244+
convention = "google"
245+
246+
[tool.ruff.lint.isort]
247+
known-first-party=[]
248+
249+
[tool.ruff.lint.mccabe]
250+
# Explicitly set default to 7, so we can later lower this threshold once all 6+ issues are solved.
251+
max-complexity = 7

0 commit comments

Comments
 (0)