multisem wafer 60 and 61 first pass

README.md

@@ -2,16 +2,16 @@
 Pipeline scripts and tools for reconstructing Electron Microscopy volumes.
 
 ### Setup
-1. If necessary, [install miniconda](https://docs.conda.io/en/latest/miniconda.html).
+1. If necessary, [install miniforge](https://github.com/conda-forge/miniforge?tab=readme-ov-file#install).
 
 ### Create Environment and Install 
 ```bash
 git clone https://github.com/JaneliaSciComp/EM_recon_pipeline.git
 
 cd EM_recon_pipeline
 
-conda env create -f janelia_emrp.environment.yml
-conda activate janelia_emrp
+conda env create -f janelia_emrp_3_12.environment.yml
+conda activate janelia_emrp_3_12
 
 poetry install
 ```

janelia_emrp_3_12.environment.yml

@@ -0,0 +1,10 @@
+name: janelia_emrp_3_12
+
+channels:
+  - default
+  - conda-forge
+
+dependencies:
+  - python=3.12
+  - urllib3=1.26
+  - poetry

pyproject.toml

@@ -1,22 +1,26 @@
 [tool.poetry]
 name = "janelia-emrp"
-version = "0.1.0"
+version = "0.2.0"
 description = "Pipeline scripts and tools for reconstructing Electron Microscopy volumes."
 authors = ["Eric Trautman <[email protected]>"]
 license = 'BSD-3-Clause'
+package-mode = false
 
 [tool.poetry.dependencies]
-python = "^3.9"
-fibsem-tools = "0.3.1"
+python = "^3.12"
 render-python = {git = "https://github.com/AllenInstitute/render-python.git"}
 dask-janelia = "^0.1.4"
 bokeh = "^2.3.3"
 h5py = "^3.6.0"
 pytest = "^7.1.1"
-xarray-multiscale = "0.3.1"
-pydantic = "^1.9.0"
-numpy = "1.21.0"
-scikit-image = "0.19.3"
+xarray-multiscale = "0.3.3"
+pydantic = "^2.7.0"
+numpy = "1.26.4"
+scikit-image = "0.22.0"
+matplotlib = "*"
+google-cloud-storage = "2.19.0"
+opencv-python = "4.10.0.84"
+basicpy = "1.2.0"
 
 [tool.poetry.dev-dependencies]
 

src/python/janelia_emrp/match/delete_match_pairs.py

@@ -106,11 +106,10 @@ def main(arg_list: List[str]):
     try:
         main(sys.argv[1:])
         # main(["--render_host", "em-services-1.int.janelia.org:8080",
-        #       "--match_owner", "hess_wafer_53c",
-        #       "--match_collections", "s002_m395_match",
+        #       "--match_owner", "hess_wafers_60_61",
+        #       "--match_collections", "w60_s360_r00_d20_gc_match",
         #       "--pair_type", "within",
-        #       "--min_keep_weight", "0.20",
-        #       "--explain"])
+        #       "--min_keep_weight", "0.20"])
     except Exception as e:
         # ensure exit code is a non-zero value when Exception occurs
         traceback.print_exc()

src/python/janelia_emrp/msem/field_of_view_layout.py

@@ -1,27 +1,6 @@
+from dataclasses import dataclass
 from typing import List, Dict, Tuple
 
-# Column group ordering for standard 7 mFOV layout:
-#     03
-#   01  06
-#     04
-#   02  07
-#     05
-SEVEN_MFOV_COLUMN_GROUPS = [[1, 2], [3, 4, 5], [6, 7]]
-
-
-# Column group ordering for standard 19 mFOV layout:
-#       08
-#     04  13
-#   01  09  17
-#     05  14
-#   02  10  18
-#     06  15
-#   03  11  19
-#     07  16
-#       12
-NINETEEN_MFOV_COLUMN_GROUPS = [[1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11, 12], [13, 14, 15, 16], [17, 18, 19]]
-
-
 # Mapping for 91 sFOV spiral from center of mFOV pattern that inserts empty/gap columns to improve visualization.
 #
 #     column:  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19  20
@@ -58,41 +37,211 @@
     "082": (10, 5), "083": (10, 7), "084": (10, 9), "085": (10, 11), "086": (10, 13), "087": (10, 15)
 }
 
+# Identifies a "space" mfov in a column that is needed for positioning but is not a real mfov.
+SPACE_MFOV_NUMBER = -1
+
+# For a layout with MFOVs that each contain 91 SFOVs,
+# the rough difference in y between the centers of two adjacent mfovs in the same column.
+# Needs to be big enough to allow for the largest difference across all mfovs (but not too big).
+NINETY_ONE_SFOV_ADJACENT_MFOV_DELTA_Y = 17000  # 16334 - 16428
+
+@dataclass
+class MFovPosition:
+    mfov_number: int
+    center_sfov_x: int
+    center_sfov_y: int
+
+@dataclass
+class MfovLayoutColumn:
+    mfov_positions: List[MFovPosition]
+
+    def size(self):
+        return len(self.mfov_positions)
+
+    def to_mfov_number_list(self) -> List[int]:
+        return [mfov_position.mfov_number for mfov_position in self.mfov_positions]
+
+    def to_y_list(self) -> List[int]:
+        return [mfov_position.center_sfov_y for mfov_position in self.mfov_positions]
+
+@dataclass
+class MfovColumnGroup:
+    layout_columns: List[MfovLayoutColumn]
+
+    def max_number_of_mfovs_in_column(self):
+        return max([column.size() for column in self.layout_columns])
+
+    def to_list_of_mfov_number_lists(self) -> List[List[int]]:
+        return [column.to_mfov_number_list() for column in self.layout_columns]
+
+    def to_list_of_y(self) -> List[List[int]]:
+        return [column.to_y_list() for column in self.layout_columns]
+
+    def print_mfov_info(self):
+        list_of_mfov_number_lists = self.to_list_of_mfov_number_lists()
+        max_mfovs = self.max_number_of_mfovs_in_column()
+        print("\nmfov columns:")
+        for i in range(len(list_of_mfov_number_lists)):
+            n_list = list_of_mfov_number_lists[i]
+            offset = " " * int((max_mfovs - len(n_list)) * 6 / 2)
+            print(f"  column {i:2d} ({len(n_list):2d} mfovs): {offset} {[str(n).zfill(2) for n in n_list]}")
+
+    def print_mfov_y_values(self):
+        print("\nmfov y values:")
+        for c_list in self.to_list_of_y():
+            print(f"  {c_list}")
+
+def build_mfov_column_group(mfov_position_list: List[MFovPosition],
+                            delta_y_for_one_row: int) -> MfovColumnGroup:
+
+    if len(mfov_position_list) == 0:
+        raise RuntimeError("mfov_position_list must not be empty")
+    elif delta_y_for_one_row < 1:
+        raise RuntimeError("delta_y_for_one_row must be positive")
+
+    sorted_by_mfov_number = sorted(mfov_position_list, key=lambda mp: mp.mfov_number)
+    previous_mfov = sorted_by_mfov_number[0]
+    layout_column = MfovLayoutColumn(mfov_positions=[previous_mfov])
+    column_group = MfovColumnGroup(layout_columns=[layout_column])
+    min_y = previous_mfov.center_sfov_y
+    max_y = previous_mfov.center_sfov_y
+
+    for current_mfov in sorted_by_mfov_number[1:]:
+
+        delta_x = current_mfov.center_sfov_x - previous_mfov.center_sfov_x
+        delta_y = current_mfov.center_sfov_y - previous_mfov.center_sfov_y
+
+        if delta_x < 0 and delta_y < 0:
+            raise RuntimeError(f"mfov positions must be sorted top-to-bottom, left-to-right: "
+                               f"mfov {current_mfov} to {previous_mfov} has delta_x {delta_x} and delta_y {delta_y}")
+
+        elif delta_x < delta_y:  # same column
+
+            # add "space" mfovs as needed to column
+            y = previous_mfov.center_sfov_y
+            while delta_y > delta_y_for_one_row:
+                y = y + delta_y_for_one_row
+                layout_column.mfov_positions.append(MFovPosition(SPACE_MFOV_NUMBER, current_mfov.center_sfov_x, y))
+                delta_y = delta_y - delta_y_for_one_row
+
+            # add current mfov to column
+            layout_column.mfov_positions.append(current_mfov)
+
+        else:  # new column
+            layout_column = MfovLayoutColumn(mfov_positions=[current_mfov])
+            column_group.layout_columns.append(layout_column)
+
+        min_y = min(min_y, current_mfov.center_sfov_y)
+        max_y = max(max_y, current_mfov.center_sfov_y)
+
+        previous_mfov = current_mfov
+
+    # ensure we have an odd number of columns
+    column_count = len(column_group.layout_columns)
+    added_empty_column = False
+    if column_count % 2 == 0:
+        delta_x = column_group.layout_columns[0].mfov_positions[0].center_sfov_x - column_group.layout_columns[1].mfov_positions[0].center_sfov_x
+        last_mfov = column_group.layout_columns[-1].mfov_positions[-1]
+        space_x = last_mfov.center_sfov_x + delta_x
+        space_y = last_mfov.center_sfov_y
+        column_group.layout_columns.append(MfovLayoutColumn(mfov_positions=[MFovPosition(SPACE_MFOV_NUMBER, space_x, space_y)]))
+        added_empty_column = True
+
+    middle_column_index = int(column_count / 2)
+    previous_column = None
+
+    # add empty mfovs to columns so that all columns have similar number of mfovs (within 1)
+    for column_index in range(len(column_group.layout_columns)):
+        column = column_group.layout_columns[column_index]
+        filled_mfov_positions = []
+
+        top_mfov_position = column.mfov_positions[0]
+        x = top_mfov_position.center_sfov_x
+        delta_y = top_mfov_position.center_sfov_y - min_y
+        number_of_top_spaces = int(delta_y / delta_y_for_one_row)
+
+        for j in range(number_of_top_spaces):
+            y = top_mfov_position.center_sfov_y - ((number_of_top_spaces - j) * delta_y_for_one_row)
+            filled_mfov_positions.append(MFovPosition(SPACE_MFOV_NUMBER, x, y))
+
+        filled_mfov_positions.extend(column.mfov_positions)
+
+        bottom_mfov_position = column.mfov_positions[-1]
+        x = bottom_mfov_position.center_sfov_x
+        delta_y = max_y - bottom_mfov_position.center_sfov_y
+        number_of_bottom_spaces = int(delta_y / delta_y_for_one_row)
+        for j in range(number_of_bottom_spaces):
+            y = bottom_mfov_position.center_sfov_y + ((j+1) * delta_y_for_one_row)
+            filled_mfov_positions.append(MFovPosition(SPACE_MFOV_NUMBER, x, y))
+
+        if previous_column is not None:
+            x = filled_mfov_positions[0].center_sfov_x
+            top_y = filled_mfov_positions[0].center_sfov_y
+            bottom_y = filled_mfov_positions[-1].center_sfov_y
+            if column_index <= middle_column_index:
+                while top_y > previous_column.mfov_positions[0].center_sfov_y:
+                    top_y = top_y - delta_y_for_one_row
+                    filled_mfov_positions.insert(0, MFovPosition(SPACE_MFOV_NUMBER, x, top_y))
+                while bottom_y < previous_column.mfov_positions[-1].center_sfov_y:
+                    bottom_y = bottom_y + delta_y_for_one_row
+                    filled_mfov_positions.append(MFovPosition(SPACE_MFOV_NUMBER, x, bottom_y))
+            elif len(previous_column.mfov_positions) > 1:
+                while top_y > previous_column.mfov_positions[1].center_sfov_y:
+                    top_y = top_y - delta_y_for_one_row
+                    filled_mfov_positions.insert(0, MFovPosition(SPACE_MFOV_NUMBER, x, top_y))
+                while bottom_y < previous_column.mfov_positions[-2].center_sfov_y:
+                    bottom_y = bottom_y + delta_y_for_one_row
+                    filled_mfov_positions.append(MFovPosition(SPACE_MFOV_NUMBER, x, bottom_y))
+
+        column.mfov_positions = filled_mfov_positions
+        previous_column = column
+
+    if added_empty_column:
+        column_group.layout_columns.pop()
+
+    return column_group
 
 class FieldOfViewLayout:
     def __init__(self,
-                 mfov_column_groups: List[List[int]],
+                 mfov_column_group: MfovColumnGroup,
                  sfov_index_name_to_row_col: Dict[str, Tuple[int, int]]):
 
-        self.mfov_column_groups = mfov_column_groups
+        self.mfov_column_group = mfov_column_group
         self.sfov_index_name_to_row_col = sfov_index_name_to_row_col
-        self.mfov_name_to_offsets = {}
-
-        max_number_of_mfovs_in_column = 0
-        for column_group in mfov_column_groups:
-            max_number_of_mfovs_in_column = max(max_number_of_mfovs_in_column, len(column_group))
+        self.mfov_number_to_offsets = {}
 
+        max_number_of_mfovs_in_column = mfov_column_group.max_number_of_mfovs_in_column()
         column_group_index = 0
-        for column_group in mfov_column_groups:
+        for layout_column in mfov_column_group.layout_columns:
+            #    1    2    3    4    5    6    7    8    9   10   11   12   13   14   15   16
+            # -:-- -:-- -:-- -:-- 0:72 -:-- 0:71 -:-- 0:70 -:-- 0:69 -:-- 0:68 -:-- 0:67 7:77
             col_offset = column_group_index * 16
-            row_offset = (max_number_of_mfovs_in_column - len(column_group)) * 5
-            for mfov_number in column_group:
-                mfov_name = f"{mfov_number:06d}"
-                self.mfov_name_to_offsets[mfov_name] = (row_offset, col_offset)
+
+            # 1:                               7:72
+            # 2:                          7:73 -:--
+            # 3:                     7:74 -:-- 7:47
+            # 4:                7:75 -:-- 7:48 -:--
+            # 5:           7:76 -:-- 7:49 -:-- 7:28
+            # 6: 0:67 7:77 -:-- 7:50 -:-- 7:29 -:--
+            row_offset = (max_number_of_mfovs_in_column - layout_column.size()) * 5
+
+            for mfov_number in layout_column.to_mfov_number_list():
+                if mfov_number >= 0:
+                    self.mfov_number_to_offsets[mfov_number] = (row_offset, col_offset)
                 row_offset = row_offset + 11
             column_group_index += 1
 
     def row_and_col(self,
-                    mfov_name: str,
-                    sfov_index_name: str) -> (int, int):
-        row_offset, col_offset = self.mfov_name_to_offsets[mfov_name]
+                    mfov_number: int,
+                    sfov_index_name: str) -> Tuple[int, int]:
+        row_offset, col_offset = self.mfov_number_to_offsets[mfov_number]
         sfov_row, sfov_col = self.sfov_index_name_to_row_col[sfov_index_name]
         return sfov_row + row_offset, sfov_col + col_offset
 
     def build_sfov_index_name_matrix(self) -> List[List[str]]:
         max_row_offset = 0
         max_col_offset = 0
-        for key, value in self.mfov_name_to_offsets.items():
+        for key, value in self.mfov_number_to_offsets.items():
             row_offset, col_offset = value
             max_row_offset = max(max_row_offset, row_offset)
             max_col_offset = max(max_col_offset, col_offset)
@@ -113,23 +262,32 @@ def build_sfov_index_name_matrix(self) -> List[List[str]]:
                 names_for_row.append("      ")
             sfov_index_name_matrix.append(names_for_row)
 
-        for mfov_name in sorted(self.mfov_name_to_offsets.keys()):
+        for mfov_number in sorted(self.mfov_number_to_offsets.keys()):
             for sfov_index_name in sorted(self.sfov_index_name_to_row_col.keys()):
-                row, col = self.row_and_col(mfov_name, sfov_index_name)
-                sfov_index_name_matrix[row][col] = f"{mfov_name[-2:]}:{sfov_index_name}"
+                row, col = self.row_and_col(mfov_number, sfov_index_name)
+                sfov_index_name_matrix[row][col] = f"{mfov_number:02d}:{sfov_index_name}"
 
         return sfov_index_name_matrix
 
+    def print_sfov_index_name_matrix(self):
+        sfov_index_name_matrix = self.build_sfov_index_name_matrix()
+        for i in range(0, len(sfov_index_name_matrix)):
+            names_for_row = sfov_index_name_matrix[i]
+            for j in range(0, len(names_for_row)):
+                print(names_for_row[j], end=" ")
+            print("")
 
 def main():
-    layout = FieldOfViewLayout(SEVEN_MFOV_COLUMN_GROUPS, NINETY_ONE_SFOV_NAME_TO_ROW_COL)
-    # layout = FieldOfViewLayout(NINETEEN_MFOV_COLUMN_GROUPS, NINETY_ONE_SFOV_NAME_TO_ROW_COL)
-    sfov_index_name_matrix = layout.build_sfov_index_name_matrix()
-    for i in range(0, len(sfov_index_name_matrix)):
-        names_for_row = sfov_index_name_matrix[i]
-        for j in range(0, len(names_for_row)):
-            print(names_for_row[j], end=" ")
-        print("")
+    column_group = build_mfov_column_group(
+        [
+            MFovPosition( 0, 10, 100), MFovPosition( 1, 10, 110),
+            MFovPosition( 2, 20,  45), MFovPosition( 3, 20,  55), MFovPosition( 4, 20,  65), MFovPosition(5, 20,  75), MFovPosition(6, 20,  85), MFovPosition(7, 20,  95), MFovPosition(8, 20, 105), MFovPosition(9, 20, 115),
+            MFovPosition(10, 30,  50), MFovPosition(11, 30,  60), MFovPosition(12, 30,  70)
+        ],
+        10)
+
+    layout = FieldOfViewLayout(column_group, NINETY_ONE_SFOV_NAME_TO_ROW_COL)
+    layout.print_sfov_index_name_matrix()
 
 
 if __name__ == '__main__':