review

src/python/janelia_emrp/msem/ingestion_ibeammsem/path.py

@@ -40,12 +40,14 @@ def get_sfov_path(slab_path: Path, mfov: int, sfov: int) -> Path:
         The microscope numbering is 1-indexed:
             the sfov with ID=0 points to sfov_001.png
     """
-    return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"sfov_{sfov+1:03}.png"
+    return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"sfov_{sfov + 1:03}.png"
 
 
 def get_thumbnail_path(slab_path: Path, mfov: int, sfov: int) -> Path:
     """Returns the path of the SFOV thumbnail given the slab path."""
-    return get_mfov_path(slab_path=slab_path, mfov=mfov) / f"thumbnail_{sfov+1:03}.png"
+    return (
+        get_mfov_path(slab_path=slab_path, mfov=mfov) / f"thumbnail_{sfov + 1:03}.png"
+    )
 
 
 def get_image_paths(

src/python/janelia_emrp/msem/ingestion_ibeammsem/review/review.py

@@ -0,0 +1,127 @@
+"""Review of IBEAM-MSEM data.
+
+Before ingesting IBEAM-MSEM data,
+the IBEAM-MSEM or data acquisition operators
+may add a final review to document non-nominal items.
+
+This review is contained in an array in the xlog.
+
+The granularity level of the review array is XDim.MFOV.
+Every MFOV has a set of one or more flags that describe the MFOV,
+e.g. {Flag.NOMINAL}
+or {Flag.DISTORTION_Y_LINEAR_MILD, Flag.OFFSET_SALVAGEABLE}.
+
+The ingestion operators take actions about MFOVs.
+The actions are defined in ReviewAction,
+e.g., ReviewAction.USE or ReviewAction.NO_Z_DROP.
+
+The mapping from a set of ReviewFlags to ReviewActions is called a review strategy.
+It defines what ingestion action to take depending on the flags,
+e.g. we ReviewAction.USE MFOVs with the flag {Flag.NOMINAL}
+e.g. we ReviewAction.DROP_NO_Z MFOVs with the flags {Flag.TEST, Flag.DISTORTION_Y_LINEAR_MILD}
+
+We can define different strategies.
+Strategies are labeled with integers.
+E.g., in strategy #0, we are conservative
+    and decide to use only ReviewFlag.NOMINAL data
+    and drop all the rest.
+E.g., in strategy #1, we are less conservative and ingest more edge cases.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+from janelia_emrp.msem.ingestion_ibeammsem.review.reviewstrategy import REVIEW_STRATEGY
+from janelia_emrp.msem.ingestion_ibeammsem.review.reviewerror import (
+    FlagSetWithNoActionError,
+)
+from janelia_emrp.msem.ingestion_ibeammsem.xdim import XDim
+from janelia_emrp.msem.ingestion_ibeammsem.xvar import XVar
+from janelia_emrp.msem.ingestion_ibeammsem.review.reviewflag import ReviewFlag
+
+if TYPE_CHECKING:
+    import xarray as xr
+    from janelia_emrp.msem.ingestion_ibeammsem.review.reviewaction import ReviewAction
+
+
+def get_review_flag(
+    xlog: xr.Dataset,
+    scan: int | list[int] | np.ndarray | slice = slice(0, None),
+    slab: int | list[int] | np.ndarray | slice = slice(0, None),
+    mfov: int | list[int] | np.ndarray | slice = slice(0, None),
+) -> xr.DataArray:
+    """Returns the review flags of MFOVs.
+
+    Omit a dimension argument to select all items of the dimension.
+    E.g. get_review_flag(scan=12)
+        returns the review flags of all MFOVs in all slabs in scan 12.
+    """
+    return xlog[XVar.REVIEW].sel(scan=scan, slab=slab, mfov=mfov)
+
+
+def get_review_action(
+    review_flag: xr.DataArray, scan: int, slab: int, mfov: int, review_strategy: int
+) -> ReviewAction:
+    """Gets the review action of an MFOV given a review flag array.
+
+    The review flag array must contain the MFOV of interest.
+
+    Possible use:
+    review_flag_slab = get_review_flag(slab=0).load()
+    for scan in scans:
+        for mfov in mfovs:
+            action = get_review_action(review_flag_slab, scan=scan, slab=0, mfov=mfov)
+            if action is Action.USE:
+                ...
+            elif action is Action.WITH_Z_MASK:
+                ...
+    """
+    review_flag_mfov = review_flag.expand_dims(
+        tuple({XDim.SCAN, XDim.SLAB, XDim.MFOV} - set(review_flag.dims))
+    ).sel(scan=scan, slab=slab, mfov=mfov)
+    key_flags: frozenset[int] = frozenset(
+        review_flag_mfov.where(review_flag_mfov)
+        .dropna(XDim.REVIEW_FLAG)[XDim.REVIEW_FLAG]
+        .values
+    )
+    if key_flags not in REVIEW_STRATEGY[review_strategy]:
+        raise FlagSetWithNoActionError(set(key_flags))
+    return REVIEW_STRATEGY[review_strategy][key_flags]
+
+
+def get_flag_sets_without_action(
+    review: xr.DataArray, review_strategy: int
+) -> list[set[ReviewFlag]]:
+    """Sets of flags that exist in the review array but miss a defined action.
+
+    When starting a new ingestion for a new dataset,
+    or when extending the ingestion to more scans,
+    we may want to check up-front what sets of review flags
+    are missing an action,
+    instead of stopping multiple times at runtime with NoActionFlagErrors.
+
+    E.g., the method could return
+        [
+            {
+                <ReviewFlag.REDEPOSITED_MATERIAL: 7>,
+                <ReviewFlag.DISTORTION_Y_NONLINEAR_MAYBE: 11>,
+            },
+            {
+                <ReviewFlag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN: 6>,
+                <ReviewFlag.DISTORTION_Y_NONLINEAR_MAYBE: 11>,
+            },
+        ]
+    We should then add two more entries in the review strategy to handle these two cases.
+    """
+    flag_patterns = np.unique(
+        review.stack(all_mfovs=tuple(set(review.dims) - {XDim.REVIEW_FLAG})).transpose(
+            "all_mfovs", ...
+        ),
+        axis=0,
+    )
+    missing_keys = {
+        frozenset(np.nonzero(flag_pattern)[0]) for flag_pattern in flag_patterns
+    } - set(REVIEW_STRATEGY[review_strategy].keys())
+    return [{ReviewFlag(flag_int) for flag_int in key} for key in missing_keys]

src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewaction.py

@@ -0,0 +1,35 @@
+"""ReviewAction."""
+
+from enum import IntEnum
+
+
+class ReviewAction(IntEnum):
+    """Action about an item during data ingestion."""
+
+    USE = 0
+    """Use the item for the ingestion.
+
+    It is the nominal action.
+    """
+    WITH_Z_MASK = 1
+    """The item would have filled a Z-slice: mask it.
+
+    WITH_Z means that a nominal image of the item is not available
+    and it would have filled a Z-slice slot.
+    """
+    WITH_Z_INPAINT = 2
+    """The item would have filled a Z-slice: inpaint it.
+
+    WITH_Z means that a nominal image of the item is not available
+    and it would have filled a Z-slice slot.
+    """
+    NO_Z_DROP = 3
+    """The item would not have filled a Z-slice: drop it.
+
+    We typically decide to NO_Z_DROP items with any of these ReviewFlag:
+        REDEPOSITED_MATERIAL
+        DEPLETED
+        NO_SAMPLE_IN_SLAB_NO_LOSS
+
+    The item is not meant to fill a Z-slice.
+    """

src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewerror.py

@@ -0,0 +1,12 @@
+"""ReviewError"""
+
+
+class ReviewError(Exception):
+    """ReviewError."""
+
+
+class FlagSetWithNoActionError(ReviewError):
+    """A set of flags is missing a defined action.
+
+    You should add a new entry in the ReviewStrategy.
+    """

src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewflag.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from enum import IntEnum
+
+
+class ReviewFlag(IntEnum):
+    """Review of the acquired data prior to ingestion.
+
+    The xarray contains a variable REVIEW.
+    This enum defines the values of this variable.
+    """
+
+    NOMINAL = 0
+    """nominal data"""
+    NO_FILE = 1
+    """the file is missing"""
+    OFFSET_SALVAGEABLE = 2
+    """data can be salvaged by applying an unusual spatial offset.
+
+    The stage can suffer from lost step events.
+    After a lost step event, for all stage move requests to (x,y)
+        the stage moves to (x + dx, y + dy) instead of (x,y)
+        with the following properties:
+            1. either dx or dy is non-zero
+            2. (dx,dy) remains constant until
+                the lost step event is fixed
+                or until another lost step event occurs.
+    In most cases, the acquisition pipeline
+        1. notices the events
+        2. fixes the issue
+        3. re-acquires affected items
+        so that the end data is not affected.
+
+    In rare cases, some of the 3 items above have failed.
+
+    We classify the symptoms at the MFOV level:
+        OFFSET_SALVAGEABLE
+        OFFSET_LOSS
+
+    The drawings below show an example slab containing 3 MFOVs.
+    The imagery of interest is represented by ABCDEFGHIJKLM.
+    The drawing on the left shows the nominal case.
+    The drawing on the right shows the problem.
+
+    MFOV #1 is labeled as OFFSET_LOSS: there is no useful data
+
+    MFOVs #2 and #3 are labeled as OFFSET_SALVAGEABLE:
+        there is useful data
+        an unusually large spatial offset is needed
+            to align the data.
+
+    The MFOVs do not cover the sample part "LM": it is a true data loss.
+
+     MFOV#1  MFOV#2  MFOV#3     MFOV#1  MFOV#2  MFOV#3 
+    +------++------++------+   +------++------++------+
+    |  ABCD||EFGHIJ||KLM   |-->|      || ABCDE||FGHIJK|LM
+    |      ||      ||      |   |      ||      ||      |
+    +------++------++------+   +------++------++------+
+    """
+    OFFSET_LOSS = 3
+    """There is no data to salvage from this item.
+
+    See OFFSET_SALVAGEABLE.
+    OFFSET_LOSS corresponds to MFOV#1 in the example drawing.
+    """
+    DISTORTION_Y_LINEAR_MILD = 4
+    """data has a mild y-axis linear distortion.
+
+    Due to a failure of the scan amplifier of the microscope.
+    Applying linear transforms should fix the distortion.
+    There might be some true lost data between SFOVs
+        because they do not overlap.
+    """
+    DISTORTION_Y_LINEAR_SEVERE = 5
+    """data has a severe y-axis linear distortion."""
+    DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN = 6
+    """data has a severe y-axis linear distortion and has been retaken later."""
+    REDEPOSITED_MATERIAL = 7
+    """image of redeposited material which is not of interest.
+
+    The electron irradiation step of IBEAM-MSEM
+    produces redeposited material at the surface of slabs.
+    The redeposited material is not part of the final dataset.
+    This flag is typically present in the early scans of an experiment,
+    e.g. scans [0,1,2].
+    """
+    DEPLETED = 8
+    """IBEAM depleted the slab of material of interest.
+
+    When IBEAM depleted a slab of its material of interest
+        and IBEAM-MSEM operators determined that the slab is finished,
+        they stop acquiring data for that slab.
+    This determination is made at the slab level, e.g.,
+        one slab is still being acquired while another one is not acquired any more.
+    Several factors influence why some slabs require more scans than others.
+
+    The determination from IBEAM-MSEM operators may be too conservative:
+        more scans of a slab were acquired than necessary, e.g.,
+        starting from scan #75 onwards, the scans of a slab do not contain useful data.
+        The IBEAM-MSEM operators may, but do not have to,
+        flag the acquired data after scan #75 as DEPLETED.
+    """
+    NO_SAMPLE_IN_SLAB_NO_LOSS = 9
+    """the sample is missing from the slab but there is no lost data.
+
+    For example, when mechanical sectioning of the sample block
+    is stopped to collect on a new wafer, 
+    the first cut slab after the interruption might be partial,
+    e.g. slab #1 in drawing below.
+    That is, only one part of the slab is present
+    and the rest is physically missing.
+    The IBEAM-MSEM operator might decide
+        to still acquire the part that seems missing
+        in case there is some sample in the apparently missing slab part
+        that could not be seen in the light micrograph overview.
+
+    The missing thickness does not necessarily rebalance at the next slab,
+    intead, the missing thickness might rebalance smoothly over several slabs.
+
+    Side view of slabs:
+
+    -----A-----------A-----     slab #0 scan #0
+    -----B-----------B-----     slab #0 scan #1
+    -----C-----------C-----
+    -----D-----------D-----
+    -----E-----------E-----
+
+    -----F-----                 slab #1 is partial
+    -----G-----
+
+    -----H-----------F-----     slab #2
+    -----I-----------G-----
+    -----J-----------H-----
+               (-----I----) the missing thickness does not
+               (-----J----) necessarily rebalance at the next slab
+    """
+    TEST = 10
+    """IBEAM-MSEM operators acquired some test data."""
+    DISTORTION_Y_NONLINEAR_MAYBE = 11
+    """It is possible that data has a y-axis non-linear distortion.
+
+    Flag likely specific to Janelia wafers #60/#61 only.
+    A nonlinear distortion along the y axis affected some MFOVs during some scans.
+    The distortion occurred approximately randomly,
+    but is restricted to specific scans only.
+    All SFOVs of an affected MFOV show the same distortion.
+    """

src/python/janelia_emrp/msem/ingestion_ibeammsem/review/reviewstrategy.py

@@ -0,0 +1,56 @@
+"""Strategy for ingestion depending on review.
+
+A strategy is a mapping from a set of ReviewFlags to a ReviewAction:
+dict[frozenset[Flag], Action]
+
+An MFOV might have the flags
+    {
+        Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN,
+        Flag.DISTORTION_Y_NONLINEAR_MAYBE,
+    }.
+A strategy defines an action for MFOVs with such a set of flags.
+"""
+
+from janelia_emrp.msem.ingestion_ibeammsem.review.reviewflag import ReviewFlag as Flag
+from janelia_emrp.msem.ingestion_ibeammsem.review.reviewaction import (
+    ReviewAction as Action,
+)
+
+fset = frozenset
+
+REVIEW_STRATEGY: dict[int, dict[frozenset[Flag], Action]] = {
+    0: {
+        # Action.USE
+        fset({Flag.NOMINAL}): Action.USE,
+        fset({Flag.DISTORTION_Y_LINEAR_MILD}): Action.USE,
+        fset({Flag.DISTORTION_Y_NONLINEAR_MAYBE}): Action.USE,
+        # Action.WITH_Z_MASK
+        fset({Flag.NO_FILE}): Action.WITH_Z_MASK,
+        fset({Flag.OFFSET_LOSS}): Action.WITH_Z_MASK,
+        fset({Flag.OFFSET_SALVAGEABLE}): Action.WITH_Z_MASK,
+        fset({Flag.DISTORTION_Y_LINEAR_SEVERE}): Action.WITH_Z_MASK,
+        # Action.NO_Z_DROP
+        fset({Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN}): Action.NO_Z_DROP,
+        fset(
+            {
+                Flag.DISTORTION_Y_LINEAR_SEVERE_LATER_RETAKEN,
+                Flag.DISTORTION_Y_NONLINEAR_MAYBE,
+            }
+        ): Action.NO_Z_DROP,
+        fset({Flag.REDEPOSITED_MATERIAL}): Action.NO_Z_DROP,
+        fset(
+            {Flag.REDEPOSITED_MATERIAL, Flag.DISTORTION_Y_NONLINEAR_MAYBE}
+        ): Action.NO_Z_DROP,
+        fset(
+            {
+                Flag.REDEPOSITED_MATERIAL,
+                Flag.DISTORTION_Y_NONLINEAR_MAYBE,
+                Flag.NO_SAMPLE_IN_SLAB_NO_LOSS,
+            }
+        ): Action.NO_Z_DROP,
+        fset({Flag.DEPLETED}): Action.NO_Z_DROP,
+        fset({Flag.NO_SAMPLE_IN_SLAB_NO_LOSS}): Action.NO_Z_DROP,
+        fset({Flag.TEST}): Action.NO_Z_DROP,
+    },
+    # 1: add your custom strategy
+}