Deploy for release 0.10.1 from 3d2a0f2

Author: rcannood

CHANGELOG.md

@@ -1,3 +1,19 @@
+# openpipelines 0.10.1
+
+## MINOR CHANGES
+
+* `integration/scvi_leiden`: Expose hvg selection argument `--var_input` (#543, PR #547).
+
+## BUG FIXES
+
+* `integration/bbknn_leiden`: Set leiden clustering parameter to multiple (#542, PR #545).
+
+* `integration/scvi_leiden`: Fix component name in Viash config (PR #547).
+
+* `integration/harmony_leiden`: Pass `--uns_neighbors` argument `umap` (PR #548).
+
+* Add workaround for bug where resources aren't available when using Nextflow fusion by including `setup_logger`, `subset_vars` and `compress_h5mu` in the script itself (PR #549).
+
 # openpipelines 0.10.0
 
 ## BREAKING CHANGES

src/annotate/popv/script.py

@@ -52,7 +52,22 @@ def mps_is_available():
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 use_gpu = cuda_is_available() or mps_is_available()

src/cluster/leiden/script.py

@@ -18,7 +18,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 logger.info("Reading %s.", par["input"])

src/compression/compress_h5mu/script.py

@@ -9,7 +9,59 @@
 ### VIASH END
 
 sys.path.append(meta["resources_dir"])
-from compress_h5mu import compress_h5mu
+# START TEMPORARY WORKAROUND compress_h5mu
+# reason: resources aren't available when using Nextflow fusion
+# from compress_h5mu import compress_h5mu
+from h5py import File as H5File
+from h5py import Group, Dataset
+from pathlib import Path
+from typing import Union, Literal
+from functools import partial
+
+
+def compress_h5mu(input_path: Union[str, Path], 
+                output_path: Union[str, Path], 
+                compression: Union[Literal['gzip'], Literal['lzf']]):
+    input_path, output_path = str(input_path), str(output_path)
+
+    def copy_attributes(in_object, out_object):
+        for key, value in in_object.attrs.items():
+            out_object.attrs[key] = value
+
+    def visit_path(output_h5: H5File,
+                   compression: Union[Literal['gzip'], Literal['lzf']], 
+                   name: str, object: Union[Group, Dataset]):
+            if isinstance(object, Group):
+                new_group = output_h5.create_group(name)
+                copy_attributes(object, new_group)
+            elif isinstance(object, Dataset):
+                # Compression only works for non-scalar Dataset objects
+                # Scalar objects dont have a shape defined
+                if not object.compression and object.shape not in [None, ()]: 
+                    new_dataset = output_h5.create_dataset(name, data=object, compression=compression)
+                    copy_attributes(object, new_dataset)
+                else:
+                    output_h5.copy(object, name)
+            else:
+                raise NotImplementedError(f"Could not copy element {name}, "
+                                          f"type has not been implemented yet: {type(object)}")
+
+    with H5File(input_path, 'r') as input_h5, H5File(output_path, 'w', userblock_size=512) as output_h5:
+        copy_attributes(input_h5, output_h5)
+        input_h5.visititems(partial(visit_path, output_h5, compression))
+
+    with open(input_path, "rb") as input_bytes:
+        # Mudata puts metadata like this in the first 512 bytes:
+        # MuData (format-version=0.1.0;creator=muon;creator-version=0.2.0)
+        # See mudata/_core/io.py, read_h5mu() function
+        starting_metadata = input_bytes.read(100)
+        # The metadata is padded with extra null bytes up until 512 bytes
+        truncate_location = starting_metadata.find(b"\x00")
+        starting_metadata = starting_metadata[:truncate_location]
+    with open(output_path, "br+") as f:
+        nbytes = f.write(starting_metadata)
+        f.write(b"\0" * (512 - nbytes)) 
+# END TEMPORARY WORKAROUND compress_h5mu
 
 if __name__ == "__main__":
     compress_h5mu(par["input"], par["output"], compression=par["compression"])

src/convert/from_10xh5_to_h5mu/script.py

@@ -16,7 +16,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 logger.info("Reading %s.", par["input"])

src/convert/from_10xmtx_to_h5mu/script.py

@@ -10,7 +10,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 logger.info("Reading %s.", par["input"])

src/convert/from_cellranger_multi_to_h5mu/script.py

@@ -17,7 +17,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 POSSIBLE_LIBRARY_TYPES = ('vdj_t', 'vdj_b', 'vdj_t_gd', 'count')

src/convert/from_h5ad_to_h5mu/script.py

@@ -13,7 +13,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 assert len(par["input"]) == len(par["modality"]), "Number of input files should be the same length as the number of modalities"

src/convert/from_h5mu_to_h5ad/script.py

@@ -11,7 +11,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 # TODO: Merge modalities into one layer

src/correction/cellbender_remove_background/script.py

@@ -67,7 +67,22 @@
 ## VIASH END
 
 sys.path.append(meta["resources_dir"])
-from setup_logger import setup_logger
+# START TEMPORARY WORKAROUND setup_logger
+# reason: resources aren't available when using Nextflow fusion
+# from setup_logger import setup_logger
+def setup_logger():
+    import logging
+    from sys import stdout
+
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    console_handler = logging.StreamHandler(stdout)
+    logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
+    console_handler.setFormatter(logFormatter)
+    logger.addHandler(console_handler)
+
+    return logger
+# END TEMPORARY WORKAROUND setup_logger
 logger = setup_logger()
 
 from helper import anndata_from_h5