File writer

CHANGELOG.rst

@@ -8,9 +8,10 @@ Changelog for Stingray Controller
 Added:
 ^^^^^^
 - Handling for new firmware error reporting
+- Stim waiting state to add spinner to stim start/stop button
 
 
-0.3.0 (unreleased)
+0.3.0 (2023-07-11)
 ------------------
 
 Added:
@@ -26,8 +27,8 @@ Fixed:
 ^^^^^^
 - Error message for when:
 
-    - The instrument's firmware is incompatible with the software
-    - An error occurs during software install
+  - The instrument's firmware is incompatible with the software
+  - An error occurs during software install
 
 
 0.2.0 (2023-06-05)

controller/build.py

@@ -32,5 +32,7 @@
 
 for output in cmd.get_outputs():
     relative_extension = os.path.relpath(output, cmd.build_lib)
-    # TODO delete the .so file it if already exists
-    shutil.copyfile(output, os.path.join("src", relative_extension))
+    dest_path = os.path.join("src", relative_extension)
+    if os.path.exists(dest_path):
+        os.remove(dest_path)
+    shutil.copyfile(output, dest_path)

controller/pyproject.toml

@@ -15,10 +15,11 @@ httpx = "0.24.1"
 aioserial = "1.3.1"
 Cython = "0.29.34"
 immutabledict = "2.2.3"
+jsonschema = "4.17.3"  # Tanner (7/7/23): pinning this to avoid issues with pyinstaller. Can probably remove this dependency entirely once labware-domain-models is removed
 labware-domain-models = "0.3.1"
 numpy = "1.23.5"  # pinned for pulse3d
 # psutil = "5.9.4"
-pulse3d = "0.33.5"
+pulse3d = "0.33.10"
 pyserial = "3.5"
 semver = "2.13.0"
 stdlib-utils = "0.5.2"
@@ -33,7 +34,7 @@ aioconsole = "0.6.0"
 freezegun = "1.2.2"
 # pefile = "2023.2.7" # Tanner (2/24/23): this must be explicitly specified so that it will be included in a Windows build environment
 pre-commit = "3.1.1"
-pyinstaller = "5.8.0"
+pyinstaller = "5.13.0"
 pytest = "7.2.1"
 pytest-asyncio = "0.20.3"
 pytest-cov = "4.0.0"
@@ -42,7 +43,6 @@ pytest-profiling = "1.7.0"
 pytest-randomly = "3.12.0"
 pytest-timeout = "2.1.0"
 # pywin32-ctypes = "0.2.0" # Tanner (2/24/23): this must be explicitly specified so that it will be included in a Windows build environment
-requests = "2.28.2"
 
 [build-system]
 requires = ["poetry-core", "setuptools", "Cython", "numpy"]
@@ -68,7 +68,7 @@ ignore_missing_imports = true
 
 
 [tool.pytest.ini_options]
-addopts = "--cov=controller --cov-report html --cov-branch --cov-report term-missing:skip-covered --cov-fail-under=46"
+addopts = "--cov=controller --cov-report html --cov-branch --cov-report term-missing:skip-covered --cov-fail-under=53"
 markers = [
     "only_run_in_ci",  # marks tests that only need to be run during full Continuous Integration testing environment (select to run with '--full-ci' if conftest.py configured)
     "slow",  # marks tests that take a bit longer to run, but can be run during local development (select to run with '--include-slow-tests' if conftest.py configured)

controller/src/controller/constants.py

@@ -11,16 +11,21 @@
 from labware_domain_models import LabwareDefinition
 
 
+# TODO organize this file
+
 # General
 CURRENT_SOFTWARE_VERSION = "REPLACETHISWITHVERSIONDURINGBUILD"
 COMPILED_EXE_BUILD_TIMESTAMP = "REPLACETHISWITHTIMESTAMPDURINGBUILD"
 SOFTWARE_RELEASE_CHANNEL = "REPLACETHISWITHRELEASECHANNELDURINGBUILD"
 
 DEFAULT_SERVER_PORT_NUMBER = 4565
 
+CURRENT_RECORDING_FILE_VERSION = "2.0.0"
+
 NUM_WELLS = 24
 GENERIC_24_WELL_DEFINITION = LabwareDefinition(row_count=4, column_count=6)
 
+RECORDINGS_SUBDIR = "recordings"
 FW_UPDATE_SUBDIR = "firmware_updates"
 
 AuthTokens = namedtuple("AuthTokens", ["access", "refresh"])
@@ -30,6 +35,7 @@
 VALID_CREDENTIAL_TYPES = frozenset(AuthCreds._fields)
 VALID_CONFIG_SETTINGS = frozenset(ConfigSettings._fields)
 
+BARCODE_LEN = 12
 # TODO try replacing all immutabledicts with enums
 BARCODE_HEADERS: immutabledict[str, str] = immutabledict({"plate_barcode": "ML", "stim_barcode": "MS"})
 ALL_VALID_BARCODE_HEADERS = frozenset(BARCODE_HEADERS.values())
@@ -53,17 +59,23 @@
 
 class SystemStatuses(Enum):
     # boot up states
-    SERVER_INITIALIZING_STATE = uuid.UUID("04471bcf-1a00-4a0d-83c8-4160622f9a25")
-    SERVER_READY_STATE = uuid.UUID("8e24ef4d-2353-4e9d-aa32-4346126e73e3")
-    SYSTEM_INITIALIZING_STATE = uuid.UUID("d2e3d386-b760-4c9a-8b2d-410362ff11c4")
-    CHECKING_FOR_UPDATES_STATE = uuid.UUID("04fd6f6b-ee9e-4656-aae4-0b9584791f36")
+    SERVER_INITIALIZING = uuid.UUID("04471bcf-1a00-4a0d-83c8-4160622f9a25")
+    SERVER_READY = uuid.UUID("8e24ef4d-2353-4e9d-aa32-4346126e73e3")
+    SYSTEM_INITIALIZING = uuid.UUID("d2e3d386-b760-4c9a-8b2d-410362ff11c4")
+    CHECKING_FOR_UPDATES = uuid.UUID("04fd6f6b-ee9e-4656-aae4-0b9584791f36")
+    # initial set up states
+    CALIBRATION_NEEDED = uuid.UUID("009301eb-625c-4dc4-9e92-1a4d0762465f")
+    CALIBRATING = uuid.UUID("43c08fc5-ca2f-4dcd-9dff-5e9324cb5dbf")
     # normal operation states
-    IDLE_READY_STATE = uuid.UUID("009301eb-625c-4dc4-9e92-1a4d0762465f")
+    IDLE_READY = uuid.UUID("b480373b-9466-4fa0-92a6-fa5f8e340d30")
+    BUFFERING = uuid.UUID("dc774d4b-6bd1-4717-b36e-6df6f1ef6cf4")
+    LIVE_VIEW_ACTIVE = uuid.UUID("9fbee58e-c6af-49a5-b2e2-5b085eead2ea")
+    RECORDING = uuid.UUID("1e3d76a2-508d-4c99-8bf5-60dac5cc51fe")
     # updating states
-    UPDATES_NEEDED_STATE = uuid.UUID("d6dcf2a9-b6ea-4d4e-9423-500f91a82a2f")
-    DOWNLOADING_UPDATES_STATE = uuid.UUID("b623c5fa-af01-46d3-9282-748e19fe374c")
-    INSTALLING_UPDATES_STATE = uuid.UUID("19c9c2d6-0de4-4334-8cb3-a4c7ab0eab00")
-    UPDATES_COMPLETE_STATE = uuid.UUID("31f8fbc9-9b41-4191-8598-6462b7490789")
+    UPDATES_NEEDED = uuid.UUID("d6dcf2a9-b6ea-4d4e-9423-500f91a82a2f")
+    DOWNLOADING_UPDATES = uuid.UUID("b623c5fa-af01-46d3-9282-748e19fe374c")
+    INSTALLING_UPDATES = uuid.UUID("19c9c2d6-0de4-4334-8cb3-a4c7ab0eab00")
+    UPDATES_COMPLETE = uuid.UUID("31f8fbc9-9b41-4191-8598-6462b7490789")
 
 
 class StimulationStates(Enum):
@@ -209,6 +221,24 @@ class SerialCommPacketTypes(IntEnum):
 GOING_DORMANT_HANDSHAKE_TIMEOUT_CODE = 0
 
 
+# Magnetometer configuration
+SERIAL_COMM_SENSOR_AXIS_LOOKUP_TABLE: immutabledict[str, dict[str, int]] = immutabledict(
+    {
+        "A": {"X": 0, "Y": 1, "Z": 2},
+        "B": {"X": 3, "Y": 4, "Z": 5},
+        "C": {"X": 6, "Y": 7, "Z": 8},
+    }
+)
+NUM_CHANNELS_PER_MAG_SENSOR = 3
+NUM_MAG_SENSORS_PER_WELL = 3
+NUM_MAG_DATA_CHANNELS_PER_WELL = NUM_CHANNELS_PER_MAG_SENSOR * NUM_MAG_SENSORS_PER_WELL
+
+DEFAULT_MAG_DATA_CHANNEL = SERIAL_COMM_SENSOR_AXIS_LOOKUP_TABLE["A"]["Z"]
+DEFAULT_MAG_SAMPLING_PERIOD = 10000  # valid as of 4/12/23
+NUM_MAG_DATA_PACKETS_PER_SECOND = MICRO_TO_BASE_CONVERSION // DEFAULT_MAG_SAMPLING_PERIOD
+
+NUM_INITIAL_MAG_PACKETS_TO_DROP = 2
+
 # Stimulation
 STIM_MAX_ABSOLUTE_CURRENT_MICROAMPS = int(100e3)
 STIM_MAX_ABSOLUTE_VOLTAGE_MILLIVOLTS = int(1.2e3)
@@ -307,3 +337,10 @@ class StimProtocolStatuses(IntEnum):
 STIM_WELL_IDX_TO_MODULE_ID: immutabledict[int, int] = immutabledict(
     {well_idx: module_id for module_id, well_idx in STIM_MODULE_ID_TO_WELL_IDX.items()}
 )
+
+
+# Recording
+CALIBRATION_RECORDING_DUR_SECONDS = 30
+
+FILE_WRITER_BUFFER_SIZE_SECONDS = 30
+FILE_WRITER_BUFFER_SIZE_MILLISECONDS = FILE_WRITER_BUFFER_SIZE_SECONDS * MICRO_TO_BASE_CONVERSION

controller/src/controller/exceptions.py

@@ -13,6 +13,10 @@ class WebsocketCommandError(Exception):
     pass
 
 
+class WebsocketCommandNoOpException(Exception):
+    pass
+
+
 class ElectronControllerVersionMismatchError(Exception):
     pass
 

controller/src/controller/main.py

@@ -4,11 +4,9 @@
 
 import argparse
 import asyncio
-import hashlib
 import logging
 import os
 import platform
-import socket
 import sys
 from typing import Any
 import uuid
@@ -28,6 +26,7 @@
 from .subsystems.cloud_comm import CloudComm
 from .subsystems.instrument_comm import InstrumentComm
 from .utils.aio import wait_tasks_clean
+from .utils.generic import get_hash_of_computer_name
 from .utils.logging import configure_logging
 from .utils.logging import redact_sensitive_info_from_path
 from .utils.state_management import SystemStateManager
@@ -73,18 +72,26 @@ async def main(command_line_args: list[str]) -> None:
         system_state_manager = SystemStateManager()
         await system_state_manager.update(initialize_system_state(parsed_args, log_file_id))
 
-        queues = create_system_queues()
+        comm_queues = create_system_comm_queues()
+        data_queues = create_system_data_queues()
 
         # create subsystems
-        system_monitor = SystemMonitor(system_state_manager, queues)
+        system_monitor = SystemMonitor(system_state_manager, comm_queues)
         server = Server(
-            system_state_manager.get_read_only_copy, queues["to"]["server"], queues["from"]["server"]
+            system_state_manager.get_read_only_copy,
+            comm_queues["to"]["server"],
+            comm_queues["from"]["server"],
         )
         instrument_comm_subsystem = InstrumentComm(
-            queues["to"]["instrument_comm"], queues["from"]["instrument_comm"]
+            comm_queues["to"]["instrument_comm"],
+            comm_queues["from"]["instrument_comm"],
+            data_queues["main"],
+            data_queues["file_writer"],
         )
         cloud_comm_subsystem = CloudComm(
-            queues["to"]["cloud_comm"], queues["from"]["cloud_comm"], **_get_user_config_settings(parsed_args)
+            comm_queues["to"]["cloud_comm"],
+            comm_queues["from"]["cloud_comm"],
+            **_get_user_config_settings(parsed_args),
         )
 
         # future for subsystems to set if they experience an error. The server will report the error in the future to the UI
@@ -120,14 +127,24 @@ async def main(command_line_args: list[str]) -> None:
         logger.info("Program exiting")
 
 
-# TODO consider moving this to a different file
-def create_system_queues() -> dict[str, Any]:
+# TODO consider moving these two to a different file
+def create_system_comm_queues() -> dict[str, Any]:
     return {
-        direction: {subsystem: asyncio.Queue() for subsystem in ("server", "instrument_comm", "cloud_comm")}
+        direction: {
+            subsystem: asyncio.Queue()
+            for subsystem in ("server", "instrument_comm", "cloud_comm", "file_writer")
+        }
         for direction in ("to", "from")
     }
 
 
+def create_system_data_queues() -> dict[str, Any]:
+    return {
+        receiving_subsystem: asyncio.Queue()
+        for receiving_subsystem in ("file_writer", "data_analyzer", "main")
+    }
+
+
 def _parse_cmd_line_args(command_line_args: list[str]) -> dict[str, Any]:
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -178,7 +195,7 @@ def initialize_system_state(parsed_args: dict[str, Any], log_file_id: uuid.UUID)
 
     system_state = {
         # main
-        "system_status": SystemStatuses.SERVER_INITIALIZING_STATE,
+        "system_status": SystemStatuses.SERVER_INITIALIZING,
         "in_simulation_mode": False,
         "stimulation_protocol_statuses": [],
         # updating
@@ -211,8 +228,6 @@ def _log_system_info() -> None:
     uname_release = getattr(uname, "release")
     uname_version = getattr(uname, "version")
 
-    computer_name_hash = hashlib.sha512(socket.gethostname().encode(encoding="UTF-8")).hexdigest()
-
     for msg in (
         f"System: {uname_sys}",
         f"Release: {uname_release}",
@@ -224,7 +239,7 @@ def _log_system_info() -> None:
         f"Architecture: {platform.architecture()}",
         f"Interpreter is 64-bits: {sys.maxsize > 2**32}",
         f"System Alias: {platform.system_alias(uname_sys, uname_release, uname_version)}",
-        f"SHA512 digest of Computer Name {computer_name_hash}",
+        f"SHA512 digest of Computer Name {get_hash_of_computer_name()}",
     ):
         logger.info(msg)