Merge pull request #347 from Dana-Farber-AIOS/v2.1.1

v2.1.1

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 88
+extend-ignore = E203 E501
+per-file-ignores = __init__.py:F401

.github/workflows/precommit-hooks.yaml

@@ -0,0 +1,17 @@
+name: pre-commit-hooks
+
+on: [pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Linting
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files

.pre-commit-config.yaml

@@ -5,12 +5,22 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
-    -   id: trailing-whitespace
-    -   id: end-of-file-fixer
-    -   id: check-yaml
-    -   id: check-added-large-files
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+
   - repo: https://github.com/psf/black
-    rev: 21.6b0 # Replace by any tag/version: https://github.com/psf/black/tags
+    rev: 22.8.0
     hooks:
       - id: black
-        language_version: python3 # Should be a command that runs python3.6+
+
+  - repo: https://github.com/timothycrosley/isort
+    rev: 5.11.5
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 5.0.4
+    hooks:
+      - id: flake8

Dockerfile

@@ -44,7 +44,7 @@ COPY tests/ /opt/pathml/tests
 
 # install pathml and deepcell
 RUN pip3 install pip==21.3.1 \
-    && pip3 install numpy==1.19.5 spams==2.6.2.5 \
+    && pip3 install numpy==1.19.5 spams==2.6.2.5 protobuf==3.20.1\
     && pip3 install python-bioformats==4.0.0 deepcell /opt/pathml/ pytest
 
 # run tests to verify container

docs/source/conf.py

@@ -12,13 +12,15 @@
 
 import os
 import sys
+from datetime import datetime
+
 sys.path.insert(0, os.path.abspath('../../'))
 
 
 # -- Project information -----------------------------------------------------
 
 project = 'PathML'
-copyright = '2021, Dana-Farber Cancer Institute and Weill Cornell Medicine'
+copyright = f'{datetime.now().year}, Dana-Farber Cancer Institute and Weill Cornell Medicine'
 author = 'Jacob Rosenthal et al.'
 
 about = {}

environment.yml

@@ -23,6 +23,7 @@ dependencies:
     - pip:
         - python-bioformats==4.0.0
         - python-javabridge==4.0.0
+        - protobuf==3.20.1
         - deepcell==0.11.0
         - opencv-contrib-python==4.5.3.56
         - openslide-python==1.1.2

pathml/__init__.py

@@ -3,9 +3,9 @@
 License: GNU GPL 2.0
 """
 
-from .core import *
 from . import datasets as ds
 from . import ml
 from . import preprocessing as pp
 from ._logging import PathMLLogger
 from ._version import __version__
+from .core import *  # noqa: F403

pathml/_logging.py

@@ -3,10 +3,11 @@
 License: GNU GPL 2.0
 """
 
-from loguru import logger
 import functools
 import sys
 
+from loguru import logger
+
 
 class PathMLLogger:
     """

pathml/core/__init__.py

@@ -4,16 +4,16 @@
 """
 
 from .masks import Masks
-from .slide_backends import OpenSlideBackend, BioFormatsBackend, DICOMBackend
+from .slide_backends import BioFormatsBackend, DICOMBackend, OpenSlideBackend
 from .slide_data import (
-    SlideData,
+    CODEXSlide,
     HESlide,
+    IHCSlide,
     MultiparametricSlide,
+    SlideData,
     VectraSlide,
-    CODEXSlide,
-    IHCSlide,
 )
 from .slide_dataset import SlideDataset
+from .slide_types import SlideType, types
 from .tile import Tile
 from .tiles import Tiles
-from .slide_types import SlideType, types

pathml/core/h5managers.py

@@ -3,15 +3,14 @@
 License: GNU GPL 2.0
 """
 
-import itertools
 import os
 import tempfile
-from collections import OrderedDict
 
 import anndata
-from loguru import logger
 import h5py
 import numpy as np
+from loguru import logger
+
 import pathml.core
 import pathml.core.masks
 import pathml.core.tile
@@ -35,10 +34,10 @@ def __init__(self, h5path=None, slidedata=None):
         if h5path:
             assert (
                 not slidedata
-            ), f"if creating h5pathmanager from h5path, slidedata should not be required"
+            ), "if creating h5pathmanager from h5path, slidedata should not be required"
             assert check_valid_h5path_format(
                 h5path
-            ), f"h5path must conform to .h5path standard, see documentation"
+            ), "h5path must conform to .h5path standard, see documentation"
             # copy h5path into self.h5
             for ds in h5path.keys():
                 if ds in ["fields", "masks", "tiles"]:
@@ -52,19 +51,19 @@ def __init__(self, h5path=None, slidedata=None):
                         )
 
         else:
-            assert slidedata, f"must pass slidedata object to create h5path"
+            assert slidedata, "must pass slidedata object to create h5path"
             # initialize h5path file hierarchy
             # fields
             fieldsgroup = self.h5.create_group("fields")
             # name, shape, labels
             fieldsgroup.attrs["name"] = slidedata.name
             fieldsgroup.attrs["shape"] = slidedata.slide.get_image_shape()
-            labelsgroup = self.h5["fields"].create_group("labels")
+            self.h5["fields"].create_group("labels")
             if slidedata.labels:
                 for key, label in slidedata.labels.items():
                     self.h5["fields/labels"].attrs[key] = label
             # slidetype
-            slidetypegroup = self.h5["fields"].create_group("slide_type")
+            self.h5["fields"].create_group("slide_type")
             if slidedata.slide_type:
                 for key, val in slidedata.slide_type.asdict().items():
                     self.h5["fields/slide_type"].attrs[key] = val
@@ -75,9 +74,9 @@ def __init__(self, h5path=None, slidedata=None):
             # initialize stride with 0
             tilesgroup.attrs["tile_stride"] = b"(0, 0)"
             # masks
-            masksgroup = self.h5.create_group("masks")
+            self.h5.create_group("masks")
             # counts
-            countsgroup = self.h5.create_group("counts")
+            self.h5.create_group("counts")
 
         slide_type_dict = {
             key: val for key, val in self.h5["fields/slide_type"].attrs.items()
@@ -150,7 +149,7 @@ def add_tile(self, tile):
         if tile.masks:
             # create a group to hold tile-level masks
             if "masks" not in self.h5["tiles"][str(tile.coords)].keys():
-                masksgroup = self.h5["tiles"][str(tile.coords)].create_group("masks")
+                self.h5["tiles"][str(tile.coords)].create_group("masks")
 
             # add tile-level masks
             for key, mask in tile.masks.items():
@@ -168,7 +167,7 @@ def add_tile(self, tile):
         self.h5["tiles"][str(tile.coords)].attrs["name"] = (
             str(tile.name) if tile.name else 0
         )
-        tilelabelsgroup = self.h5["tiles"][str(tile.coords)].create_group("labels")
+        self.h5["tiles"][str(tile.coords)].create_group("labels")
         if tile.labels:
             for key, val in tile.labels.items():
                 self.h5["tiles"][str(tile.coords)]["labels"].attrs[key] = val
@@ -198,7 +197,7 @@ def get_tile(self, item):
             Tile(pathml.core.tile.Tile)
         """
         if isinstance(item, bool):
-            raise KeyError(f"invalid key, pass str or tuple")
+            raise KeyError("invalid key, pass str or tuple")
         if isinstance(item, (str, tuple)):
             item = str(item)
             if item not in self.h5["tiles"].keys():
@@ -246,7 +245,7 @@ def remove_tile(self, key):
         Remove tile from self.h5 by key.
         """
         if not isinstance(key, (str, tuple)):
-            raise KeyError(f"key must be str or tuple, check valid keys in repr")
+            raise KeyError("key must be str or tuple, check valid keys in repr")
         if str(key) not in self.h5["tiles"].keys():
             raise KeyError(f"key {key} is not in Tiles")
         del self.h5["tiles"][str(key)]
@@ -270,7 +269,7 @@ def add_mask(self, key, mask):
             raise ValueError(
                 f"key {key} already exists in 'masks'. Cannot add. Must update to modify existing mask."
             )
-        newmask = self.h5["masks"].create_dataset(key, data=mask)
+        self.h5["masks"].create_dataset(key, data=mask)
 
     def update_mask(self, key, mask):
         """
@@ -340,7 +339,7 @@ def remove_mask(self, key):
                 f"masks keys must be of type(str) but key was passed of type {type(key)}"
             )
         if key not in self.h5["masks"].keys():
-            raise KeyError(f"key is not in Masks")
+            raise KeyError("key is not in Masks")
         del self.h5["masks"][key]
 
     def get_slidetype(self):

pathml/core/masks.py

@@ -3,13 +3,10 @@
 License: GNU GPL 2.0
 """
 
-import numpy as np
-import os
-from pathlib import Path
-from collections import OrderedDict
-import h5py
 import reprlib
-from loguru import logger
+from collections import OrderedDict
+
+import numpy as np
 
 import pathml.core.h5managers
 

pathml/core/slide_backends.py

@@ -4,15 +4,11 @@
 """
 
 from io import BytesIO
-from typing import Tuple
 
 import numpy as np
 import openslide
-from loguru import logger
-import pathml.core
-import pathml.core.tile
 from javabridge.jutil import JavaException
-from pathml.utils import pil_to_rgb
+from loguru import logger
 from PIL import Image
 from pydicom.dataset import Dataset
 from pydicom.encaps import get_frame_offsets
@@ -22,14 +18,18 @@
 from pydicom.uid import UID
 from scipy.ndimage import zoom
 
+import pathml.core
+import pathml.core.tile
+from pathml.utils import pil_to_rgb
+
 try:
     import bioformats
     import javabridge
     from bioformats.metadatatools import createOMEXMLMetadata
 except ImportError:
     logger.exception("Unable to import bioformats, javabridge")
     raise Exception(
-        f"Installation of PathML not complete. Please install openjdk8, bioformats, and javabridge:\nconda install openjdk==8.0.152\npip install javabridge==1.0.19 python-bioformats==4.0.0\nFor detailed installation instructions, please see https://github.com/Dana-Farber-AIOS/pathml/"
+        "Installation of PathML not complete. Please install openjdk8, bioformats, and javabridge:\nconda install openjdk==8.0.152\npip install javabridge==1.0.19 python-bioformats==4.0.0\nFor detailed installation instructions, please see https://github.com/Dana-Farber-AIOS/pathml/"
     )
 
 
@@ -330,7 +330,8 @@ def __init__(self, filename, dtype=None):
             try:
                 self.pixel_dtype = pixel_dtype_map[ome_pixeltype]
                 logger.info(f"Found corresponding dtype: {self.pixel_dtype}")
-            except:
+            # TODO: change to specific exception
+            except Exception:
                 logger.exception("datatype from metadata not found in pixel_dtype_map")
                 raise Exception(
                     f"pixel type '{ome_pixeltype}' detected from OME metadata not recognized."
@@ -411,7 +412,7 @@ def extract_region(
                 f"input size {size} invalid. Must be a tuple of integer coordinates of len<2"
             )
         if series_as_channels:
-            logger.info(f"using series_as_channels=True")
+            logger.info("using series_as_channels=True")
             if level != 0:
                 logger.exception(
                     f"When series_as_channels=True, must use level=0. Input 'level={level}' invalid."
@@ -490,7 +491,7 @@ def extract_region(
                 f"Scaling image to [0, 1] by dividing by {(2 ** (8 * self.pixel_dtype.itemsize))}"
             )
             # then scale to [0-255] and convert to 8 bit
-            array_scaled = array_scaled * 2 ** 8
+            array_scaled = array_scaled * 2**8
             return array_scaled.astype(np.uint8)
 
     def get_thumbnail(self, size=None):
@@ -507,7 +508,7 @@ def get_thumbnail(self, size=None):
             shape = data.slide.get_image_shape()
             thumb = data.slide.get_thumbnail(size=(1000,1000, shape[2], shape[3], shape[4]))
         """
-        assert isinstance(size, (tuple, type(None))), f"Size must be a tuple of ints."
+        assert isinstance(size, (tuple, type(None))), "Size must be a tuple of ints."
         if size is not None:
             if len(size) != len(self.shape):
                 size = size + self.shape[len(size) :]
@@ -520,7 +521,7 @@ def get_thumbnail(self, size=None):
             ratio = tuple([x / y for x, y in zip(size, self.shape)])
             assert (
                 ratio[3] == 1
-            ), f"cannot interpolate between fluor channels, resampling doesn't apply, fix size[3]"
+            ), "cannot interpolate between fluor channels, resampling doesn't apply, fix size[3]"
             image_array = zoom(array, ratio)
         return image_array
 
@@ -768,7 +769,7 @@ def extract_region(self, location, size=None, level=None):
         Returns:
             np.ndarray: image at the specified region
         """
-        assert level == 0 or level is None, f"dicom does not support levels"
+        assert level == 0 or level is None, "dicom does not support levels"
         # check inputs first
         # check location
         if isinstance(location, tuple):
@@ -887,7 +888,7 @@ def generate_tiles(self, shape, stride, pad, level=0, **kwargs):
         Yields:
             pathml.core.tile.Tile: Extracted Tile object
         """
-        assert level == 0 or level is None, f"dicom does not support levels"
+        assert level == 0 or level is None, "dicom does not support levels"
         for i in range(self.n_frames):
 
             if not pad: