Merge pull request #168 from ral-facilities/DSEGOG-383-Update-Poetry.toml-for-noxfile.py

Dsegog 383 update poetry.toml for noxfile.py

Author: moonraker595

.github/ci_requirements.txt

@@ -1,2 +1,2 @@
-poetry==1.3.2
-nox==2022.1.7
+poetry==2.1.1
+nox==2025.2.9

.github/workflows/ci.yml

@@ -17,17 +17,17 @@ jobs:
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
       # Install dependencies of python-ldap
       - name: Install python-ldap dependencies
         run: |
           sudo apt-get update
           sudo apt-get install -y libsasl2-dev libldap2-dev libssl-dev
 
-      # Setup Python and environment dependencies (via cache)
-      - name: Setup Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.11
       - name: Load Pip cache
         uses: actions/cache@v3
         with:
@@ -51,7 +51,6 @@ jobs:
           echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-7.0.list
           sudo apt-get update
           sudo apt-get install -y mongodb-database-tools
-
       # Read the database name from the config file and store it in an environment variable
       - name: Get database name from ci_config.yml
         run: echo "DATABASE_NAME=$(grep database_name .github/ci_config.yml | cut -d ':' -f 2 | tr -d '[:space:]')" >> $GITHUB_ENV
@@ -60,10 +59,8 @@ jobs:
       - name: Load Poetry cache
         uses: actions/cache@v3
         with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: ${{ runner.os }}-poetry-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
-      - name: Install dependencies
-        run: poetry install --without simulated-data
+          path: ~/.cache/pypoetry
+          key: ${{ runner.os }}-poetry-3.11-${{ hashFiles('poetry.lock') }}
 
       # Setup minio and create bucket
       - name: Setup minio
@@ -78,11 +75,12 @@ jobs:
           export AWS_SECRET_ACCESS_KEY=minioadmin
           export AWS_EC2_METADATA_DISABLED=true
           aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://og-actions-test
-
       - name: Move CI config.yml to correct place
         run: cp .github/ci_config.yml operationsgateway_api/config.yml
+
       - name: Setup logging configuration
         run: cp operationsgateway_api/logging.ini.example operationsgateway_api/logging.ini
+
       - name: Create log file
         run: touch "$GITHUB_WORKSPACE/logs.log"
 
@@ -116,21 +114,15 @@ jobs:
       - name: Run Echo Ingest script
         run: poetry run python util/realistic_data/ingest_echo_data.py
 
-      - name: Load Poetry cache for Nox tests session
-        uses: actions/cache@v3
-        with:
-          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/tests*
-          key: ${{ runner.os }}-poetry-nox-tests-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
-
       # Run Nox tests session, saves and uploads a coverage report to codecov
       - name: Run Nox tests session
-        run: nox -p 3.11 -s tests -- --cov=operationsgateway_api --cov-report=xml
+        run: nox -s tests -- --cov=operationsgateway_api --cov-report=xml
+
       - name: Upload code coverage report
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-
   linting:
     runs-on: ubuntu-latest
     name: Code Linting
@@ -139,6 +131,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"
+
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
@@ -150,24 +143,24 @@ jobs:
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
-      - name: Load Poetry cache for Nox lint session
+      - name: Load Poetry cache
         uses: actions/cache@v3
         with:
-          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/lint*
-          key: ${{ runner.os }}-poetry-nox-lint-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          path: ~/.cache/pypoetry
+          key: poetry-${{ runner.os }}-3.11-${{ hashFiles('poetry.lock') }}
 
       - name: Run Nox lint session
         run: nox -s lint
 
-
   formatting:
     runs-on: ubuntu-latest
-    name: Code Formatting
+    name: Black Formatting
     steps:
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"
+
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
@@ -179,16 +172,15 @@ jobs:
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
-      - name: Load Poetry cache for Nox black session
+      - name: Load Poetry cache
         uses: actions/cache@v3
         with:
-          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/black*
-          key: ${{ runner.os }}-poetry-nox-black-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          path: ~/.cache/pypoetry
+          key: poetry-${{ runner.os }}-3.11-${{ hashFiles('poetry.lock') }}
 
       - name: Run Nox black session
         run: nox -s black -- --check --diff
 
-
   safety:
     runs-on: ubuntu-latest
     name: Dependency Safety
@@ -197,6 +189,7 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"
+
       - name: Checkout OperationsGateway API
         uses: actions/checkout@v3
 
@@ -208,11 +201,11 @@ jobs:
       - name: Install Poetry & Nox
         run: pip install -r .github/ci_requirements.txt
 
-      - name: Load Poetry cache for Nox safety session
+      - name: Load Poetry cache
         uses: actions/cache@v3
         with:
-          path: /home/runner/work/operationsgateway-api/operationsgateway-api/.nox/safety*
-          key: ${{ runner.os }}-poetry-nox-safety-3.11-${{ env.pythonLocation }}-${{ hashFiles('poetry.lock') }}
+          path: ~/.cache/pypoetry
+          key: poetry-${{ runner.os }}-3.11-${{ hashFiles('poetry.lock') }}
 
-      - name: Run Nox safety session
-        run: nox -s safety
+      - name: Run Safety Scan
+        run: nox -s safety

.gitignore

@@ -5,6 +5,10 @@ venv/
 *.log.*
 config.yml*
 !config.yml.example
+schedulued_maintenance.yml*
+!schedulued_maintenance.yml.example
+maintenance.yml*
+!maintenance.yml.example
 .vscode/
 .nox/
 .python-version

README.md

@@ -13,7 +13,8 @@ The following instructions will detail how to get a development instance up and
 ## Prerequisites
 
 1) Local development on Windows or Mac can be challenging because they lack the Linux libraries required by the API. Since the API is designed to run in a Python 3.11 environment on a Rocky 9 Linux machine, it’s best to use a Rocky 9 development VM to closely mirror the production setup.
-2) One of the dependencies used in this API [(`epac-data-sim`)](https://github.com/CentralLaserFacility/EPAC-DataSim) is a private repository, so the appropriate permissions and SSH keys need to be set up. Guidance for setting up SSH keys for the Rocky 9 VM can be found [here](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent?platform=linux).
+2) You'll need a developer bucket in echo to use the test script to prefill echo, and a local database, with data.
+3) One of the dependencies used in this API [(`epac-data-sim`)](https://github.com/CentralLaserFacility/EPAC-DataSim) is a private repository, so the appropriate permissions and SSH keys need to be set up. Guidance for setting up SSH keys for the Rocky 9 VM can be found [here](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent?platform=linux).
 
 
 ## Environment Setup
@@ -45,6 +46,8 @@ cd operationsgateway-api
 poetry env use python3.11
 # Install the dependencies
 poetry install
+# If you don't need the simulated data, or don't have access to the epac repo...
+poetry install --without simulated-data
 ```
 
 ### MongoDB:

noxfile.py

@@ -1,117 +1,48 @@
-import os
-import tempfile
-
 import nox
-from nox.command import CommandFailed
 
-# Separating Black away from the rest of the sessions
-nox.options.sessions = "lint", "safety", "tests"
+nox.options.sessions = "black", "lint", "safety", "tests"
 code_locations = "operationsgateway_api", "test", "noxfile.py", "util"
 
 
-def install_with_constraints(session, *args, **kwargs):
-    # Auto file deletion is turned off to prevent a PermissionError experienced on
-    # Windows
-    with tempfile.NamedTemporaryFile(delete=False) as requirements:
-        session.run(
-            "poetry",
-            "export",
-            "--dev",
-            "--format=requirements.txt",
-            "--without-hashes",
-            f"--output={requirements.name}",
-            external=True,
-        )
-        # Using sed to remove extras from the constraints file, thereby fixing the
-        # error "Constraints cannot have extras". A better solution might be to
-        # use `poetry install --only dev` but this requires a newer version of
-        # Poetry than we're using in our development environments. See
-        # https://github.com/jazzband/pip-tools/issues/1300#issuecomment-818122483
-        # for more info relating to the sed solution
-        sed_expression = r"s/\[.*\]//g"
-        try:
-            session.run("sed", "-i", sed_expression, requirements.name, external=True)
-        except CommandFailed:
-            # Try running the Mac version of the command
-            session.run(
-                "sed",
-                "-i",
-                "",
-                sed_expression,
-                requirements.name,
-                external=True,
-            )
-
-        session.install(f"--constraint={requirements.name}", *args, **kwargs)
-
-        try:
-            # Due to delete=False, the file must be deleted manually
-            requirements.close()
-            os.unlink(requirements.name)
-        except IOError:
-            session.log("Error: The temporary requirements file could not be closed")
-
-
-@nox.session(reuse_venv=True)
+@nox.session(python=False)
 def black(session):
+    # Use Poetry’s existing virtual environment
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
+    # The dependencies should already be installed,
+    # but explicitly setting in case they're not
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
     args = session.posargs
-    install_with_constraints(session, "black")
-    session.run("black", *code_locations, *args, external=True)
+    session.run("poetry", "run", "black", *code_locations, *args, external=True)
 
 
-@nox.session(reuse_venv=True)
+@nox.session(python=False)
 def lint(session):
     args = session.posargs or code_locations
-    install_with_constraints(
-        session,
-        "flake8",
-        "flake8-black",
-        "flake8-broken-line",
-        "flake8-bugbear",
-        "flake8-builtins",
-        "flake8-commas",
-        "flake8-comprehensions",
-        "flake8-import-order",
-        "flake8-logging-format",
-        "pep8-naming",
-    )
-    session.run("flake8", *args)
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
+    session.run("poetry", "run", "flake8", *args, external=True)
 
 
-@nox.session(reuse_venv=True)
+@nox.session(python=False)
 def safety(session):
-    install_with_constraints(session, "safety")
-    with tempfile.NamedTemporaryFile(delete=False) as requirements:
-        session.run(
-            "poetry",
-            "export",
-            "--dev",
-            "--format=requirements.txt",
-            "--without-hashes",
-            f"--output={requirements.name}",
-            external=True,
-        )
-        session.run(
-            "safety",
-            "check",
-            f"--file={requirements.name}",
-            "--full-report",
-            # jinja2 report widely disputed as not valid, no fix available:
-            # https://github.com/pallets/jinja/issues/1994
-            "--ignore",
-            "70612",
-        )
-
-        try:
-            # Due to delete=False, the file must be deleted manually
-            requirements.close()
-            os.unlink(requirements.name)
-        except IOError:
-            session.log("Error: The temporary requirements file could not be closed")
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
+    session.run("poetry", "install", "--without", "simulated-data", external=True)
+    # Can't fix 70790 because epac data sim uses it
+    session.run(
+        "poetry",
+        "run",
+        "safety",
+        "check",
+        "--full-report",
+        "--ignore",
+        "70790",
+        external=True,
+    )
 
 
-@nox.session(python=["3.11"], reuse_venv=True)
+@nox.session(python=False)
 def tests(session):
     args = session.posargs
+    session.env["POETRY_VIRTUALENVS_CREATE"] = "false"
     session.run("poetry", "install", "--without", "simulated-data", external=True)
-    session.run("pytest", *args)
+    session.run("poetry", "run", "pytest", *args, external=True)