Add the components to test the postgres database on github actions

.github/workflows/ci.yml

@@ -29,6 +29,16 @@ jobs:
           - 3306:3306
         options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3
 
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
+        ports:
+          - 5432:5432
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+
     steps:
       - uses: actions/checkout@v4
       - name: Use Python ${{ matrix.python-version }}
@@ -57,7 +67,7 @@ jobs:
           docker run --detach --name rabbitmq -p 127.0.0.1:5672:5672 -p 127.0.0.1:15672:15672 test-rabbitmq
           docker container list -a
 
-      - name: Get database
+      - name: Get ispyb database
         uses: actions/download-artifact@v4
         with:
           name: database
@@ -74,7 +84,7 @@ jobs:
           mysql-version: "11.3"
           auto-start: false
 
-      - name: Set up test database
+      - name: Set up test ipsyb database
         run: |
           set -eu
           cp ".github/workflows/config/my.cnf" .my.cnf
@@ -103,6 +113,12 @@ jobs:
         run: wget -t 10 -w 1 http://127.0.0.1:15672 -O -
 
       - name: Run tests
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_DB: murfey_test_db
+          POSTGRES_PASSWORD: psql_pwd
+          POSTGRES_USER: psql_user
         run: |
           export ISPYB_CREDENTIALS=".github/workflows/config/ispyb.cfg"
           PYTHONDEVMODE=1 pytest -v -ra --cov=murfey --cov-report=xml --cov-branch

src/murfey/workflows/spa/flush_spa_preprocess.py

@@ -5,10 +5,10 @@
 from PIL import Image
 from sqlalchemy.exc import NoResultFound
 from sqlmodel import Session, select
-from werkzeug.utils import secure_filename
 
 from murfey.server import _murfey_id, _transport_object, sanitise
 from murfey.server.api.auth import MurfeySessionID
+from murfey.util import secure_path
 from murfey.util.config import get_machine_config, get_microscope
 from murfey.util.db import (
     AutoProcProgram,
@@ -72,11 +72,8 @@ def register_grid_square(
         else:
             # mock up response so that below still works
             gs_ispyb_response = {"success": False, "return_value": None}
-        secured_grid_square_image_path = secure_filename(grid_square_params.image)
-        if (
-            secured_grid_square_image_path
-            and Path(secured_grid_square_image_path).is_file()
-        ):
+        secured_grid_square_image_path = secure_path(Path(grid_square_params.image))
+        if secured_grid_square_image_path and secured_grid_square_image_path.is_file():
             jpeg_size = Image.open(secured_grid_square_image_path).size
         else:
             jpeg_size = (0, 0)
@@ -98,7 +95,7 @@ def register_grid_square(
             thumbnail_size_x=grid_square_params.thumbnail_size_x or jpeg_size[0],
             thumbnail_size_y=grid_square_params.thumbnail_size_y or jpeg_size[1],
             pixel_size=grid_square_params.pixel_size,
-            image=secured_grid_square_image_path,
+            image=str(secured_grid_square_image_path),
         )
     murfey_db.add(grid_square)
     murfey_db.commit()
@@ -124,8 +121,8 @@ def register_foil_hole(
             f"Foil hole {sanitise(str(foil_hole_params.name))} could not be registered as grid square {sanitise(str(gs_name))} was not found"
         )
         return None
-    secured_foil_hole_image_path = secure_filename(foil_hole_params.image)
-    if foil_hole_params.image and Path(secured_foil_hole_image_path).is_file():
+    secured_foil_hole_image_path = secure_path(Path(foil_hole_params.image))
+    if foil_hole_params.image and secured_foil_hole_image_path.is_file():
         jpeg_size = Image.open(secured_foil_hole_image_path).size
     else:
         jpeg_size = (0, 0)
@@ -188,7 +185,7 @@ def register_foil_hole(
             thumbnail_size_x=foil_hole_params.thumbnail_size_x or jpeg_size[0],
             thumbnail_size_y=foil_hole_params.thumbnail_size_y or jpeg_size[1],
             pixel_size=foil_hole_params.pixel_size,
-            image=secured_foil_hole_image_path,
+            image=str(secured_foil_hole_image_path),
         )
     murfey_db.add(foil_hole)
     murfey_db.commit()

tests/__init__.py

@@ -0,0 +1,10 @@
+import os
+
+from sqlmodel import create_engine
+
+url = (
+    f"postgresql+psycopg2://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}"
+    f"@{os.environ['POSTGRES_HOST']}:{os.environ['POSTGRES_PORT']}/{os.environ['POSTGRES_DB']}"
+)
+
+engine = create_engine(url)

tests/conftest.py

@@ -0,0 +1,17 @@
+import pytest
+from sqlmodel import Session
+
+from murfey.util.db import Session as MurfeySession
+from murfey.util.db import clear, setup
+from tests import engine, url
+
+
+@pytest.fixture
+def start_postgres():
+    clear(url)
+    setup(url)
+
+    murfey_session = MurfeySession(id=2, name="cm12345-6")
+    with Session(engine) as murfey_db:
+        murfey_db.add(murfey_session)
+        murfey_db.commit()

tests/workflows/spa/test_flush_spa_preprocess.py

@@ -0,0 +1,151 @@
+from unittest import mock
+
+from sqlmodel import Session, select
+
+from murfey.util.db import DataCollectionGroup, GridSquare
+from murfey.util.models import GridSquareParameters
+from murfey.workflows.spa import flush_spa_preprocess
+from tests import engine
+
+
+@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
+def test_register_grid_square_update_add_locations(mock_transport, start_postgres):
+    """Test the updating of an existing grid square"""
+    # Create a grid square to update
+    grid_square = GridSquare(
+        id=1,
+        name=101,
+        session_id=2,
+        tag="session_tag",
+    )
+    with Session(engine) as murfey_db:
+        murfey_db.add(grid_square)
+        murfey_db.commit()
+
+    # Parameters to update with
+    new_parameters = GridSquareParameters(
+        tag="session_tag",
+        x_location=1.1,
+        y_location=1.2,
+        x_stage_position=1.3,
+        y_stage_position=1.4,
+    )
+
+    # Run the registration
+    with Session(engine) as murfey_db:
+        flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
+
+    # Check this would have updated ispyb
+    mock_transport.do_update_grid_square.assert_called_with(1, new_parameters)
+
+    # Confirm the database was updated
+    with Session(engine) as murfey_db:
+        grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
+    assert grid_square_final_parameters.x_location == new_parameters.x_location
+    assert grid_square_final_parameters.y_location == new_parameters.y_location
+    assert (
+        grid_square_final_parameters.x_stage_position == new_parameters.x_stage_position
+    )
+    assert (
+        grid_square_final_parameters.y_stage_position == new_parameters.y_stage_position
+    )
+
+
+@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
+def test_register_grid_square_update_add_nothing(mock_transport, start_postgres):
+    """Test the updating of an existing grid square, but with nothing to update with"""
+    # Create a grid square to update
+    grid_square = GridSquare(
+        id=1,
+        name=101,
+        session_id=2,
+        tag="session_tag",
+        x_location=0.1,
+        y_location=0.2,
+        x_stage_position=0.3,
+        y_stage_position=0.4,
+    )
+    with Session(engine) as murfey_db:
+        murfey_db.add(grid_square)
+        murfey_db.commit()
+
+    # Parameters to update with
+    new_parameters = GridSquareParameters(tag="session_tag")
+
+    # Run the registration
+    with Session(engine) as murfey_db:
+        flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
+
+    # Check this would have updated ispyb
+    mock_transport.do_update_grid_square.assert_called_with(1, new_parameters)
+
+    # Confirm the database was not updated
+    with Session(engine) as murfey_db:
+        grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
+    assert grid_square_final_parameters.x_location == 0.1
+    assert grid_square_final_parameters.y_location == 0.2
+    assert grid_square_final_parameters.x_stage_position == 0.3
+    assert grid_square_final_parameters.y_stage_position == 0.4
+
+
+@mock.patch("murfey.workflows.spa.flush_spa_preprocess._transport_object")
+def test_register_grid_square_insert_with_ispyb(
+    mock_transport, start_postgres, tmp_path
+):
+    # Create a data collection group for lookups
+    grid_square = DataCollectionGroup(
+        id=1,
+        session_id=2,
+        tag="session_tag",
+        atlas_id=90,
+    )
+    with Session(engine) as murfey_db:
+        murfey_db.add(grid_square)
+        murfey_db.commit()
+
+    # Set the ispyb return
+    mock_transport.do_insert_grid_square.return_value = {
+        "return_value": 1,
+        "success": True,
+    }
+
+    # Parameters to update with
+    new_parameters = GridSquareParameters(
+        tag="session_tag",
+        x_location=1.1,
+        y_location=1.2,
+        x_stage_position=1.3,
+        y_stage_position=1.4,
+        readout_area_x=2048,
+        readout_area_y=1024,
+        thumbnail_size_x=512,
+        thumbnail_size_y=256,
+        pixel_size=1.02,
+        image=f"{tmp_path}/image_path",
+        angle=12.5,
+    )
+
+    # Run the registration
+    with Session(engine) as murfey_db:
+        flush_spa_preprocess.register_grid_square(2, 101, new_parameters, murfey_db)
+
+    # Check this would have updated ispyb
+    mock_transport.do_insert_grid_square.assert_called_with(90, 101, new_parameters)
+
+    # Confirm the database entry was made
+    with Session(engine) as murfey_db:
+        grid_square_final_parameters = murfey_db.exec(select(GridSquare)).one()
+    assert grid_square_final_parameters.id == 1
+    assert grid_square_final_parameters.name == 101
+    assert grid_square_final_parameters.session_id == 2
+    assert grid_square_final_parameters.tag == "session_tag"
+    assert grid_square_final_parameters.x_location == 1.1
+    assert grid_square_final_parameters.y_location == 1.2
+    assert grid_square_final_parameters.x_stage_position == 1.3
+    assert grid_square_final_parameters.y_stage_position == 1.4
+    assert grid_square_final_parameters.readout_area_x == 2048
+    assert grid_square_final_parameters.readout_area_y == 1024
+    assert grid_square_final_parameters.thumbnail_size_x == 512
+    assert grid_square_final_parameters.thumbnail_size_y == 256
+    assert grid_square_final_parameters.pixel_size == 1.02
+    assert grid_square_final_parameters.image == f"{tmp_path}/image_path"