Allow for missing data in input metadata table.

Warn that the column is missing.
Add include_experiment_description to write nwb.

Author: morriscb

allensdk/__init__.py

@@ -35,7 +35,7 @@
 #
 import logging
 
-__version__ = '2.16.1'
+__version__ = '2.16.2'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_project_cache/tables/metadata_table_schemas.py

@@ -11,49 +11,68 @@
 class BehaviorSessionMetadataSchema(RaisingSchema):
     age_in_days = Int(required=True, description="Subject age")
     behavior_session_id = Int(
-        required=True,
+        required=False,
+        allow_none=True,
         description=(
             "Unique identifier for the "
             "behavior session to write into "
             "NWB format"
         ),
     )
     cre_line = String(
-        required=True, description="Genetic cre line of the subject."
+        required=False,
+        allow_none=True,
+        description="Genetic cre line of the subject."
     )
     date_of_acquisition = String(
-        required=True,
+        required=False,
+        allow_none=True,
         description=(
             "Date of acquisition of " "behavior session, in string " "format"
         ),
     )
     driver_line = List(
         String,
-        required=True,
+        required=False,
+        allow_none=True,
         cli_as_single_argument=True,
         description="Genetic driver line(s) of subject",
     )
     equipment_name = String(
-        required=True, description=("Name of the equipment used.")
+        required=False,
+        allow_none=True,
+        description=("Name of the equipment used.")
     )
     full_genotype = String(
-        required=True, description="Full genotype of subject"
+        required=False,
+        allow_none=True,
+        description="Full genotype of subject"
     )
     mouse_id = String(
-        required=True,
+        required=False,
+        allow_none=True,
         description="LabTracks ID of the subject. aka external_specimen_name.",
     )
     project_code = String(
-        rquired=True,
+        rquired=False,
+        allow_none=True,
         description="LabTracks ID of the subject. aka external_specimen_name.",
     )
     reporter_line = String(
-        required=True, description="Genetic reporter line(s) of subject"
+        required=False,
+        allow_none=True,
+        description="Genetic reporter line(s) of subject"
     )
     session_type = String(
-        required=True, description="Full name of session type."
+        required=False,
+        allow_none=True,
+        description="Full name of session type."
+    )
+    sex = String(
+        required=False,
+        allow_none=True,
+        description="Subject sex"
     )
-    sex = String(required=True, description="Subject sex")
 
     @mm.post_load
     def convert_date_time(self, data, **kwargs):

allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py

@@ -596,19 +596,28 @@ def _check_for_errant_omitted_stimulus(
             Dataframe with omitted stimulus removed from first row or if not
             found, return input_df unmodified.
         """
+
+        def safe_omitted_check(input_df: pd.Series,
+                               stimulus_block: Optional[int]):
+            if stimulus_block is not None:
+                first_row = input_df[
+                    input_df['stimulus_block'] == stim_block].iloc[0]
+            else:
+                first_row = input_df.iloc[0]
+
+            if not pd.isna(first_row["omitted"]):
+                if first_row["omitted"]:
+                    input_df = input_df.drop(first_row.name, axis=0)
+            return input_df
+
         if "omitted" in input_df.columns and len(input_df) > 0:
             if "stimulus_block" in input_df.columns:
                 for stim_block in input_df['stimulus_block'].unique():
-                    first_row = input_df[
-                        input_df['stimulus_block'] == stim_block].iloc[0]
-                    if not pd.isna(first_row["omitted"]):
-                        if first_row["omitted"]:
-                            input_df = input_df.drop(first_row.name, axis=0)
+                    input_df = safe_omitted_check(input_df=input_df,
+                                                  stimulus_block=stim_block)
             else:
-                first_row = input_df.iloc[0]
-                if not pd.isna(first_row["omitted"]):
-                    if first_row["omitted"]:
-                        input_df = input_df.drop(first_row.name, axis=0)
+                input_df = safe_omitted_check(input_df=input_df,
+                                              stimulus_block=None)
         return input_df
 
     @staticmethod

allensdk/brain_observatory/behavior/write_nwb/behavior/__main__.py

@@ -27,6 +27,9 @@ def run(self):
             nwb_filepath=bs_id_dir / f"behavior_session_{bs_id}.nwb",
             skip_metadata=self.args["skip_metadata_key"],
             skip_stim=self.args["skip_stimulus_file_key"],
+            include_experiment_description=self.args[
+                'include_experiment_description'
+            ]
         )
         logging.info("File successfully created")
 
@@ -43,6 +46,7 @@ def write_behavior_nwb(
         nwb_filepath: Path,
         skip_metadata: List[str],
         skip_stim: List[str],
+        include_experiment_description=True
     ) -> str:
         """Load and write a BehaviorSession as NWB.
 
@@ -61,6 +65,8 @@ def write_behavior_nwb(
             List of metadata keys to skip when comparing data.
         skip_stim : list of str
             List of stimulus file keys to skip when comparing data.
+        include_experiment_description : bool
+            If True, include experiment description in NWB file.
 
         Returns
         -------
@@ -79,7 +85,11 @@ def write_behavior_nwb(
             session_data=behavior_session_metadata,
             serializer=BehaviorSession,
         )
-        nwb_writer.write_nwb(skip_metadata=skip_metadata, skip_stim=skip_stim)
+        nwb_writer.write_nwb(
+            skip_metadata=skip_metadata,
+            skip_stim=skip_stim,
+            include_experiment_description=include_experiment_description
+        )
 
         return str(nwb_filepath)
 

allensdk/brain_observatory/behavior/write_nwb/behavior/schemas.py

@@ -1,4 +1,6 @@
 import marshmallow as mm
+from warnings import warn
+import pandas as pd
 from allensdk.brain_observatory.argschema_utilities import (
     InputFile,
     RaisingSchema,
@@ -18,6 +20,7 @@
     OutputDir,
     OutputFile,
     String,
+    Bool
 )
 
 
@@ -54,29 +57,81 @@ class Meta:
         required=True,
         description="Path of output.json to be written",
     )
+    include_experiment_description = Bool(
+        required=False,
+        description="If True, include experiment description in NWB file.",
+        default=True
+    )
 
     def _get_behavior_metadata(self, bs_row):
         """ """
         behavior_session_metadata = {}
 
-        behavior_session_metadata["age_in_days"] = bs_row["age_in_days"]
-        behavior_session_metadata["cre_line"] = bs_row["cre_line"]
-        behavior_session_metadata["date_of_acquisition"] = bs_row[
-            "date_of_acquisition"
-        ]
-        behavior_session_metadata["driver_line"] = sorted(
-            bs_row["driver_line"]
+        behavior_session_metadata["age_in_days"] = self._retrieve_value(
+            bs_row=bs_row, column_name="age_in_days"
+        )
+        behavior_session_metadata["cre_line"] = self._retrieve_value(
+            bs_row=bs_row, column_name="cre_line"
+        )
+        behavior_session_metadata["date_of_acquisition"] = self._retrieve_value(  # noqa: E501
+            bs_row=bs_row, column_name="date_of_acquisition"
+        )
+        behavior_session_metadata["driver_line"] = self._retrieve_value(
+            bs_row=bs_row, column_name="driver_line"
+        )
+        behavior_session_metadata["equipment_name"] = self._retrieve_value(
+            bs_row=bs_row, column_name="equipment_name"
+        )
+        behavior_session_metadata["full_genotype"] = self._retrieve_value(
+            bs_row=bs_row, column_name="full_genotype"
+        )
+        behavior_session_metadata["mouse_id"] = self._retrieve_value(
+            bs_row=bs_row, column_name="mouse_id"
+        )
+        behavior_session_metadata["project_code"] = self._retrieve_value(
+            bs_row=bs_row, column_name="project_code"
+        )
+        behavior_session_metadata["reporter_line"] = self._retrieve_value(
+            bs_row=bs_row, column_name="reporter_line"
+        )
+        behavior_session_metadata["session_type"] = self._retrieve_value(
+            bs_row=bs_row, column_name="session_type"
+        )
+        behavior_session_metadata["sex"] = self._retrieve_value(
+            bs_row=bs_row,
+            column_name="sex"
         )
-        behavior_session_metadata["equipment_name"] = bs_row["equipment_name"]
-        behavior_session_metadata["full_genotype"] = bs_row["full_genotype"]
-        behavior_session_metadata["mouse_id"] = bs_row["mouse_id"]
-        behavior_session_metadata["project_code"] = bs_row["project_code"]
-        behavior_session_metadata["reporter_line"] = bs_row["reporter_line"]
-        behavior_session_metadata["session_type"] = bs_row["session_type"]
-        behavior_session_metadata["sex"] = bs_row["sex"]
 
         return behavior_session_metadata
 
+    def _retrieve_value(self, bs_row: pd.Series, column_name: str):
+        """Pull a column safely, return None otherwise.
+
+        Parameters
+        ----------
+        bs_row : pd.Series
+            Row of a BehaviorSessionTable
+        column_name : str
+            Name of column to retrieve
+
+        Returns
+        -------
+        value : object
+            Value of column_name in bs_row, or None if column_name is not in
+            bs_row
+        """
+        if column_name not in bs_row.index:
+            warn(f"Warning, {column_name} not in metadata table. Unless this "
+                 "has been added to the inputs skip_metadata_key or "
+                 "skip_stimulus_file_key, creating the NWB file "
+                 "may fail.")
+            return None
+        else:
+            value = bs_row[column_name]
+            if isinstance(value, list):
+                value = sorted(value)
+            return value
+
 
 class BehaviorInputSchema(BaseInputSchema):
     behavior_session_id = Int(

allensdk/brain_observatory/behavior/write_nwb/nwb_writer_utils.py

@@ -13,6 +13,7 @@ def _update_session(
         self,
         lims_session: BehaviorSession,
         ophys_experiment_ids: Optional[List[int]] = None,
+        **kwargs
     ) -> BehaviorSession:
         """Call session methods to update certain values within the session.
 

allensdk/brain_observatory/behavior/write_nwb/ophys/__main__.py

@@ -29,6 +29,9 @@ def run(self):
             nwb_filepath=oe_id_dir / f"behavior_ophys_experiment_{oe_id}.nwb",
             skip_metadata=self.args["skip_metadata_key"],
             skip_stim=self.args["skip_stimulus_file_key"],
+            include_experiment_description=self.args[
+                'include_experiment_description'
+            ]
         )
         logging.info("File successfully created")
 
@@ -43,6 +46,7 @@ def write_experiment_nwb(
         nwb_filepath: Path,
         skip_metadata: List[str],
         skip_stim: List[str],
+        include_experiment_description=True
     ) -> str:
         """Load and write a BehaviorOphysExperiment as NWB.
 
@@ -61,6 +65,8 @@ def write_experiment_nwb(
             List of metadata keys to skip when comparing data.
         skip_stim : list of str
             List of stimulus file keys to skip when comparing data.
+        include_experiment_description : bool
+            If True, include experiment description in NWB file.
 
         Returns
         -------
@@ -83,6 +89,7 @@ def write_experiment_nwb(
             ophys_experiment_ids=self.args["ophys_container_experiment_ids"],
             skip_metadata=skip_metadata,
             skip_stim=skip_stim,
+            include_experiment_description=include_experiment_description
         )
 
         return str(nwb_filepath)