Updates to allensdk for MultiscopeSignalNoise and TaskTrainedNetworksMultiscope data

allensdk/brain_observatory/behavior/data_files/stimulus_file.py

@@ -27,13 +27,31 @@
 """
 
 
+STIMULUS_FILE_SESSION_QUERY_TEMPLATE = """
+    SELECT
+        wkf.storage_directory || wkf.filename AS stim_file
+    FROM
+        well_known_files wkf
+    WHERE
+        wkf.attachable_id = {ophys_session_id}
+        AND wkf.attachable_type = 'OphysSession'
+        AND wkf.well_known_file_type_id IN (
+            SELECT id
+            FROM well_known_file_types
+            WHERE name = 'StimulusPickle');
+"""
+
+
 def from_json_cache_key(cls, dict_repr: dict):
     return hashkey(json.dumps(dict_repr))
 
 
 def from_lims_cache_key(cls, db, behavior_session_id: int):
     return hashkey(behavior_session_id)
 
+def from_lims_cache_key_ophys(cls, db, ophys_session_id: int):
+    return hashkey(ophys_session_id)
+
 
 class StimulusFile(DataFile):
     """A DataFile which contains methods for accessing and loading visual
@@ -68,6 +86,18 @@ def from_lims(
         filepath = db.fetchone(query, strict=True)
         return cls(filepath=filepath)
 
+    @classmethod
+    @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key_ophys)
+    def from_lims_for_ophys_session(
+        cls, db: PostgresQueryMixin,
+        ophys_session_id: Union[int, str]
+    ) -> "StimulusFile":
+        query = STIMULUS_FILE_SESSION_QUERY_TEMPLATE.format(
+            ophys_session_id=ophys_session_id
+        )
+        filepath = db.fetchone(query, strict=True)
+        return cls(filepath=filepath)
+
     @staticmethod
     def load_data(filepath: Union[str, Path]) -> dict:
         filepath = safe_system_path(file_name=filepath)

allensdk/brain_observatory/behavior/data_files/sync_file.py

@@ -24,6 +24,19 @@
     AND oe.id = {ophys_experiment_id};
 """
 
+SYNC_FILE_SESSION_QUERY_TEMPLATE = """
+    SELECT wkf.storage_directory || wkf.filename AS sync_file
+    FROM ophys_experiments oe
+    JOIN ophys_sessions os ON oe.ophys_session_id = os.id
+    JOIN well_known_files wkf ON wkf.attachable_id = os.id
+    JOIN well_known_file_types wkft
+    ON wkft.id = wkf.well_known_file_type_id
+    WHERE wkf.attachable_type = 'OphysSession'
+    AND wkft.name = 'OphysRigSync'
+    AND os.id = {ophys_session_id};
+"""
+
+
 
 def from_json_cache_key(cls, dict_repr: dict):
     return hashkey(json.dumps(dict_repr))
@@ -33,6 +46,10 @@ def from_lims_cache_key(cls, db, ophys_experiment_id: int):
     return hashkey(ophys_experiment_id)
 
 
+def from_lims_cache_key_session(cls, db, ophys_session_id: int):
+    return hashkey(ophys_session_id)
+
+
 class SyncFile(DataFile):
     """A DataFile which contains methods for accessing and loading visual
     behavior stimulus *.pkl files.
@@ -65,6 +82,19 @@ def from_lims(
         filepath = db.fetchone(query, strict=True)
         return cls(filepath=filepath)
 
+
+    @classmethod
+    @cached(cache=LRUCache(maxsize=10), key=from_lims_cache_key_session)
+    def from_lims_for_ophys_session(
+        cls, db: PostgresQueryMixin,
+        ophys_session_id: Union[int, str]
+    ) -> "SyncFile":
+        query = SYNC_FILE_SESSION_QUERY_TEMPLATE.format(
+            ophys_session_id=ophys_session_id
+        )
+        filepath = db.fetchall(query, strict=True)[0]
+        return cls(filepath=filepath)
+
     @staticmethod
     def load_data(filepath: Union[str, Path]) -> dict:
         filepath = safe_system_path(file_name=filepath)

allensdk/brain_observatory/behavior/data_objects/__init__.py

@@ -1,6 +1,8 @@
 from allensdk.brain_observatory.behavior.data_objects.base._data_object_abc import DataObject  # noqa: E501, F401
 from allensdk.brain_observatory.behavior.data_objects.metadata\
     .behavior_metadata.behavior_session_id import BehaviorSessionId  # noqa: E501, F401
+from allensdk.brain_observatory.behavior.data_objects.metadata\
+    .behavior_metadata.visualcoding_session_id import VisualCodingSessionId 
 from allensdk.brain_observatory.behavior.data_objects.timestamps\
     .stimulus_timestamps.stimulus_timestamps import StimulusTimestamps  # noqa: E501, F401
 from allensdk.brain_observatory.behavior.data_objects.running_speed.running_speed import RunningSpeed  # noqa: E501, F401

allensdk/brain_observatory/behavior/data_objects/cell_specimens/cell_specimens.py

@@ -181,9 +181,10 @@ def __init__(self,
         # there seem to be cases where cell_specimen_table contains rois not in
         # events
         # See ie https://app.zenhub.com/workspaces/allensdk-10-5c17f74db59cfb36f158db8c/issues/alleninstitute/allensdk/2139     # noqa
-        events.filter_and_reorder(
-            roi_ids=cell_specimen_table['cell_roi_id'].values,
-            raise_if_rois_missing=False)
+        if events is not None:
+            events.filter_and_reorder(
+                roi_ids=cell_specimen_table['cell_roi_id'].values,
+                raise_if_rois_missing=False)
 
         self._meta = meta
         self._cell_specimen_table = cell_specimen_table
@@ -235,7 +236,8 @@ def from_lims(cls, ophys_experiment_id: int,
                   ophys_timestamps: OphysTimestamps,
                   segmentation_mask_image_spacing: Tuple,
                   exclude_invalid_rois=True,
-                  events_params: Optional[EventsParams] = None) \
+                  events_params: Optional[EventsParams] = None,
+                  include_events:  bool = True) \
             -> "CellSpecimens":
         def _get_ophys_cell_segmentation_run_id() -> int:
             """Get the ophys cell segmentation run id associated with an
@@ -297,7 +299,10 @@ def _get_events():
             ophys_timestamps=ophys_timestamps)
         dff_traces = _get_dff_traces()
         corrected_fluorescence_traces = _get_corrected_fluorescence_traces()
-        events = _get_events()
+        if include_events:
+            events = _get_events()
+        else:
+            events = None
 
         return CellSpecimens(
             cell_specimen_table=cell_specimen_table, meta=meta,

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/behavior_metadata.py

@@ -4,7 +4,7 @@
 import numpy as np
 from pynwb import NWBFile
 
-from allensdk.brain_observatory.behavior.data_files import StimulusFile
+from allensdk.brain_observatory.behavior.data_files import StimulusFile, SyncFile
 from allensdk.brain_observatory.behavior.data_objects import DataObject, \
     BehaviorSessionId
 from allensdk.brain_observatory.behavior.data_objects.base \
@@ -211,8 +211,11 @@ def from_lims(
 
         stimulus_file = StimulusFile.from_lims(
             db=lims_db, behavior_session_id=behavior_session_id.value)
-        stimulus_frame_rate = StimulusFrameRate.from_stimulus_file(
-            stimulus_file=stimulus_file)
+        # stimulus_frame_rate = StimulusFrameRate.from_stimulus_file(
+        #     stimulus_file=stimulus_file)
+        sync_file = SyncFile.from_lims_for_session(db=lims_db, behavior_session_id=behavior_session_id.value)
+        stimulus_frame_rate = StimulusFrameRate.from_sync_file(
+            sync_file=sync_file)
         session_type = SessionType.from_stimulus_file(
             stimulus_file=stimulus_file)
 

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/date_of_acquisition.py

@@ -45,6 +45,22 @@ def from_lims(
             datetime=experiment_date)
         return cls(date_of_acquisition=experiment_date)
 
+    @classmethod
+    def from_lims_for_ophys_session(
+            cls, ophys_session_id: int,
+            lims_db: PostgresQueryMixin) -> "DateOfAcquisition":
+        query = """
+                SELECT os.date_of_acquisition
+                FROM ophys_sessions os
+                WHERE os.id = {};
+                """.format(ophys_session_id)
+
+        experiment_date = lims_db.fetchone(query, strict=True)
+        experiment_date = cls._postprocess_lims_datetime(
+            datetime=experiment_date)
+        return cls(date_of_acquisition=experiment_date)
+
+
     @classmethod
     def from_nwb(cls, nwbfile: NWBFile) -> "DateOfAcquisition":
         return cls(date_of_acquisition=nwbfile.session_start_time)

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/equipment.py

@@ -43,6 +43,18 @@ def from_lims(cls, behavior_session_id: int,
         equipment_name = lims_db.fetchone(query, strict=True)
         return cls(equipment_name=equipment_name)
 
+    @classmethod
+    def from_lims_for_ophys_session(cls, ophys_session_id: int,
+                  lims_db: PostgresQueryMixin) -> "Equipment":
+        query = f"""
+            SELECT e.name AS device_name
+            FROM ophys_sessions os
+            JOIN equipment e ON e.id = os.equipment_id
+            WHERE os.id = {ophys_session_id};
+        """
+        equipment_name = lims_db.fetchone(query, strict=True)
+        return cls(equipment_name=equipment_name)
+
     @classmethod
     def from_nwb(cls, nwbfile: NWBFile) -> "Equipment":
         metadata = nwbfile.lab_meta_data['metadata']

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/foraging_id.py

@@ -30,3 +30,18 @@ def from_lims(cls, behavior_session_id: int,
         foraging_id = lims_db.fetchone(query, strict=True)
         foraging_id = uuid.UUID(foraging_id)
         return cls(foraging_id=foraging_id)
+
+    @classmethod
+    def from_lims_for_ophys_session(cls, ophys_session_id: int,
+                  lims_db: PostgresQueryMixin) -> "ForagingId":
+        query = f"""
+            SELECT
+                foraging_id
+            FROM
+                ophys_sessions
+            WHERE
+                ophys_sessions.id = {ophys_session_id};
+        """
+        foraging_id = lims_db.fetchone(query, strict=True)
+        foraging_id = uuid.UUID(foraging_id)
+        return cls(foraging_id=foraging_id)

allensdk/brain_observatory/behavior/data_objects/metadata/behavior_metadata/stimulus_frame_rate.py

@@ -1,6 +1,6 @@
 from pynwb import NWBFile
 
-from allensdk.brain_observatory.behavior.data_files import StimulusFile
+from allensdk.brain_observatory.behavior.data_files import StimulusFile, SyncFile
 from allensdk.brain_observatory.behavior.data_objects import DataObject
 from allensdk.brain_observatory.behavior.data_objects.base \
     .readable_interfaces import \
@@ -27,6 +27,15 @@ def from_stimulus_file(
         frame_rate = calc_frame_rate(timestamps=stimulus_timestamps.value)
         return cls(stimulus_frame_rate=frame_rate)
 
+    @classmethod
+    def from_sync_file(
+            cls,
+            sync_file: SyncFile) -> "StimulusFrameRate":
+        stimulus_timestamps = StimulusTimestamps.from_sync_file(
+            sync_file=sync_file)
+        frame_rate = calc_frame_rate(timestamps=stimulus_timestamps.value)
+        return cls(stimulus_frame_rate=frame_rate)
+
     @classmethod
     def from_nwb(cls, nwbfile: NWBFile) -> "StimulusFrameRate":
         metadata = nwbfile.lab_meta_data['metadata']