Merge remote-tracking branch 'origin/main' into mpes_elab_metadata

Author: rettigl

.cspell/custom-dictionary.txt

@@ -53,6 +53,7 @@ coldist
 colgrid
 COLLECTIONCOLUMN
 colname
+colorbar
 colsize
 COMPES
 coordmat

src/sed/binning/binning.py

@@ -465,6 +465,7 @@ def normalization_histogram_from_timed_dataframe(
     axis: str,
     bin_centers: np.ndarray,
     time_unit: float,
+    **kwds,
 ) -> xr.DataArray:
     """Get a normalization histogram from a timed dataframe.
 
@@ -475,17 +476,12 @@ def normalization_histogram_from_timed_dataframe(
             histogram.
         bin_centers (np.ndarray): Bin centers used for binning of the axis.
         time_unit (float): Time unit the data frame entries are based on.
+        **kwds: Additional keyword arguments passed to the bin_dataframe function.
 
     Returns:
         xr.DataArray: Calculated normalization histogram.
     """
-    bins = df[axis].map_partitions(
-        pd.cut,
-        bins=bin_centers_to_bin_edges(bin_centers),
-    )
-
-    histogram = df[axis].groupby([bins]).count().compute().values * time_unit
-    # histogram = bin_dataframe(df, axes=[axis], bins=[bin_centers]) * time_unit
+    histogram = bin_dataframe(df, axes=[axis], bins=[bin_centers], **kwds) * time_unit
 
     data_array = xr.DataArray(
         data=histogram,

src/sed/core/logging.py

@@ -11,6 +11,7 @@
 import sys
 from datetime import datetime
 from functools import wraps
+from inspect import signature
 from typing import Callable
 
 # Default log directory
@@ -43,28 +44,33 @@ def setup_logging(
     # Create base logger
     base_logger = logging.getLogger("sed")
     base_logger.setLevel(logging.DEBUG)  # Set the minimum log level for the logger
-    if set_base_handler or not base_logger.hasHandlers():
-        if base_logger.hasHandlers():
+    if set_base_handler or len(base_logger.handlers) == 0:
+        if len(base_logger.handlers):
             base_logger.handlers.clear()
 
         # Determine log file path
         if user_log_path is None:
             user_log_path = DEFAULT_LOG_DIR
-        os.makedirs(user_log_path, exist_ok=True)
-        log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log")
-
-        # Create file handler and set level to debug
-        file_handler = logging.FileHandler(log_file)
-        file_handler.setLevel(FILE_VERBOSITY)
-
-        # Create formatter for file
-        file_formatter = logging.Formatter(
-            "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d",
-        )
-        file_handler.setFormatter(file_formatter)
-
-        # Add file handler to logger
-        base_logger.addHandler(file_handler)
+        try:
+            os.makedirs(user_log_path, exist_ok=True)
+            log_file = os.path.join(user_log_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log")
+
+            # Create file handler and set level to debug
+            file_handler = logging.FileHandler(log_file)
+            file_handler.setLevel(FILE_VERBOSITY)
+
+            # Create formatter for file
+            file_formatter = logging.Formatter(
+                "%(asctime)s - %(name)s - %(levelname)s - %(message)s in %(filename)s:%(lineno)d",
+            )
+            file_handler.setFormatter(file_formatter)
+
+            # Add file handler to logger
+            base_logger.addHandler(file_handler)
+        except PermissionError:
+            logging.warning(f"Cannot create logfile in Folder {user_log_path}, disabling logfile.")
+            base_logger.addHandler(logging.NullHandler())
+            base_logger.propagate = False
 
     # create named logger
     logger = base_logger.getChild(name)
@@ -109,7 +115,11 @@ def log_call(func: Callable):
         def new_func(*args, **kwargs):
             saved_args = locals()
             args_str = ""
-            for arg in saved_args["args"][1:]:
+            for arg in (
+                saved_args["args"][1:]
+                if "self" in signature(func).parameters
+                else saved_args["args"]
+            ):
                 args_str += f"{arg}, "
             for name, arg in saved_args["kwargs"].items():
                 args_str += f"{name}={arg}, "

src/sed/core/processor.py

@@ -2283,6 +2283,8 @@ def compute(
                 )
                 # if the axes are named correctly, xarray figures out the normalization correctly
                 self._normalized = self._binned / self._normalization_histogram
+                # Set datatype of binned data
+                self._normalized.data = self._normalized.data.astype(self._binned.data.dtype)
                 self._attributes.add(
                     self._normalization_histogram.values,
                     name="normalization_histogram",
@@ -2353,36 +2355,35 @@ def get_normalization_histogram(
 
         if isinstance(df_partitions, int):
             df_partitions = list(range(0, min(df_partitions, self._dataframe.npartitions)))
+
         if use_time_stamps or self._timed_dataframe is None:
             if df_partitions is not None:
-                self._normalization_histogram = normalization_histogram_from_timestamps(
-                    self._dataframe.partitions[df_partitions],
-                    axis,
-                    self._binned.coords[axis].values,
-                    self._config["dataframe"]["columns"]["timestamp"],
-                )
+                dataframe = self._dataframe.partitions[df_partitions]
             else:
-                self._normalization_histogram = normalization_histogram_from_timestamps(
-                    self._dataframe,
-                    axis,
-                    self._binned.coords[axis].values,
-                    self._config["dataframe"]["columns"]["timestamp"],
-                )
+                dataframe = self._dataframe
+            self._normalization_histogram = normalization_histogram_from_timestamps(
+                df=dataframe,
+                axis=axis,
+                bin_centers=self._binned.coords[axis].values,
+                time_stamp_column=self._config["dataframe"]["columns"]["timestamp"],
+            )
         else:
             if df_partitions is not None:
-                self._normalization_histogram = normalization_histogram_from_timed_dataframe(
-                    self._timed_dataframe.partitions[df_partitions],
-                    axis,
-                    self._binned.coords[axis].values,
-                    self._config["dataframe"]["timed_dataframe_unit_time"],
-                )
+                timed_dataframe = self._timed_dataframe.partitions[df_partitions]
             else:
-                self._normalization_histogram = normalization_histogram_from_timed_dataframe(
-                    self._timed_dataframe,
-                    axis,
-                    self._binned.coords[axis].values,
-                    self._config["dataframe"]["timed_dataframe_unit_time"],
-                )
+                timed_dataframe = self._timed_dataframe
+            self._normalization_histogram = normalization_histogram_from_timed_dataframe(
+                df=timed_dataframe,
+                axis=axis,
+                bin_centers=self._binned.coords[axis].values,
+                time_unit=self._config["dataframe"]["timed_dataframe_unit_time"],
+                hist_mode=self.config["binning"]["hist_mode"],
+                mode=self.config["binning"]["mode"],
+                pbar=self.config["binning"]["pbar"],
+                n_cores=self.config["core"]["num_cores"],
+                threads_per_worker=self.config["binning"]["threads_per_worker"],
+                threadpool_api=self.config["binning"]["threadpool_API"],
+            )
 
         return self._normalization_histogram
 

tests/test_logging.py

@@ -0,0 +1,113 @@
+import io
+import logging
+import os
+from datetime import datetime
+
+import pytest
+
+from sed.core.logging import call_logger
+from sed.core.logging import set_verbosity
+from sed.core.logging import setup_logging
+
+
+@pytest.fixture
+def logger_():
+    logger = setup_logging("test_logger")
+    log_capture_string = io.StringIO()
+    ch = logging.StreamHandler(log_capture_string)
+    ch.setLevel(logging.DEBUG)
+    logger.addHandler(ch)
+    yield logger, log_capture_string
+
+
+def test_debug_logging(logger_):
+    logger, log_capture_string = logger_
+    logger.debug("This is a debug message")
+    assert "This is a debug message" in log_capture_string.getvalue()
+
+
+def test_info_logging(logger_):
+    logger, log_capture_string = logger_
+    logger.info("This is an info message")
+    assert "This is an info message" in log_capture_string.getvalue()
+
+
+def test_warning_logging(logger_):
+    logger, log_capture_string = logger_
+    logger.warning("This is a warning message")
+    assert "This is a warning message" in log_capture_string.getvalue()
+
+
+def test_error_logging(logger_):
+    logger, log_capture_string = logger_
+    logger.error("This is an error message")
+    assert "This is an error message" in log_capture_string.getvalue()
+
+
+def test_critical_logging(logger_):
+    logger, log_capture_string = logger_
+    logger.critical("This is a critical message")
+    assert "This is a critical message" in log_capture_string.getvalue()
+
+
+def test_set_verbosity(logger_):
+    logger, log_capture_string = logger_
+    set_verbosity(logger, verbose=True)
+    assert logger.handlers[0].level == logging.INFO
+    set_verbosity(logger, verbose=False)
+    assert logger.handlers[0].level == logging.WARNING
+
+
+def test_logger_has_base_logger(logger_):
+    logger, log_capture_string = logger_
+    assert logger.name == "sed.test_logger"
+    assert logger.parent.name == "sed"
+    assert logger.parent.parent.name == "root"
+    assert logger.parent.level == logging.DEBUG
+    assert isinstance(logger.parent.handlers[0], logging.FileHandler)
+
+
+def test_logger_creates_logfile(tmp_path):
+    logger = setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path)
+    log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log")
+    assert os.path.exists(log_file)
+    with open(log_file) as f:
+        assert f.read() == ""
+    logger.debug("This is a debug message")
+    with open(log_file) as f:
+        assert "This is a debug message" in f.read()
+
+
+def test_readonly_path(tmp_path, caplog):
+    os.chmod(tmp_path, 0o444)
+    with caplog.at_level(logging.WARNING):
+        setup_logging("test_logger", set_base_handler=True, user_log_path=tmp_path)
+    assert f"Cannot create logfile in Folder {tmp_path}, disabling logfile." in caplog.messages[0]
+    log_file = os.path.join(tmp_path, f"sed_{datetime.now().strftime('%Y-%m-%d')}.log")
+    assert not os.path.exists(log_file)
+
+
+def test_call_logger(logger_):
+    logger, log_capture_string = logger_
+
+    @call_logger(logger)
+    def test_function(test_param=None):  # noqa: ARG001
+        return
+
+    test_function(test_param=[1, 3, 5])
+    assert "test_function(test_param=[1, 3, 5])" in log_capture_string.getvalue()
+
+    test_function([1, 3, 5])
+    assert "test_function([1, 3, 5])" in log_capture_string.getvalue()
+
+    test_function()
+    assert "test_function()" in log_capture_string.getvalue()
+
+    class TestClass:
+        @call_logger(logger)
+        def test_method(self, test_param=None):  # noqa: ARG002
+            return
+
+    test_instance = TestClass()
+    test_instance.test_method(test_param=[1, 3, 5])
+    assert "test_method(test_param=[1, 3, 5])" in log_capture_string.getvalue()

tests/test_processor.py

@@ -1008,6 +1008,8 @@ def test_compute_with_normalization() -> None:
         processor.binned.data,
         (processor.normalized * processor.normalization_histogram).data,
     )
+    # check dtype
+    assert processor.normalized.dtype == processor.binned.dtype
     # bin only second dataframe partition
     result2 = processor.compute(
         bins=bins,

tutorial/10_hextof_workflow_trXPS_bam_correction.ipynb

@@ -238,7 +238,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
+    "fig,ax = plt.subplots(1,2,figsize=(6,2.25), layout='constrained')\n",
     "res.plot(robust=True, ax=ax[0], cmap='terrain')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
     "ax[0].set_title('raw')\n",
@@ -338,7 +338,7 @@
    "outputs": [],
    "source": [
     "\n",
-    "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
+    "fig,ax = plt.subplots(1,2,figsize=(6,2.25), layout='constrained')\n",
     "res_bam.sel(bam=slice(-6400,-5100)).sum('trainId').plot(ax=ax[0],robust=True, cmap='terrain')\n",
     "res_bam.sel(bam=slice(-6400,-5100)).sum('pulseId').plot(ax=ax[1],robust=True, cmap='terrain')\n",
     "plt.show()"
@@ -404,7 +404,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
+    "fig,ax = plt.subplots(1,2,figsize=(6,2.25), layout='constrained')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
     "res_corr.plot(robust=True, ax=ax[0], cmap='terrain')\n",
     "ax[0].set_title('raw')\n",
@@ -497,7 +497,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fig,ax=plt.subplots(2,2,figsize=(9,7),layout=\"constrained\")\n",
+    "fig,ax=plt.subplots(2,2,figsize=(6,6),layout=\"constrained\")\n",
     "\n",
     "plt.axes(ax[0,0])\n",
     "res.plot(cmap='terrain', robust=True)\n",
@@ -523,7 +523,7 @@
     "plt.legend(loc=1)\n",
     "plt.title(\"Sidebands with bam correction\")\n",
     "\n",
-    "fig.suptitle(f'Run {run_number}: Effect of BAM correction',fontsize='22')"
+    "fig.suptitle(f'Run {run_number}: Effect of BAM correction',fontsize='14')"
    ]
   },
   {
@@ -545,7 +545,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "sed-processor-7Jy-bAA8-py3.9",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },
@@ -559,7 +559,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.19"
+   "version": "3.12.8"
   }
  },
  "nbformat": 4,

tutorial/11_hextof_workflow_trXPS_energy_calibration_using_SB.ipynb

@@ -327,7 +327,7 @@
     "bins = [100,60]\n",
     "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")\n",
     "\n",
-    "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
+    "fig,ax = plt.subplots(1,2,figsize=(6,2.25), layout='constrained')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
     "res_corr.plot(ax=ax[0], cmap='terrain')\n",
     "ax[0].set_title('raw')\n",
@@ -443,7 +443,7 @@
     "bins = [200,60]\n",
     "res_corr = sp_44498.compute(bins=bins, axes=axes, ranges=ranges, normalize_to_acquisition_time=\"delayStage\")\n",
     "\n",
-    "fig,ax = plt.subplots(1,2,figsize=(8,3), layout='constrained')\n",
+    "fig,ax = plt.subplots(1,2,figsize=(6,2.25), layout='constrained')\n",
     "fig.suptitle(f\"Run {run_number}: W 4f, side bands\")\n",
     "res_corr.plot(ax=ax[0], cmap='terrain')\n",
     "ax[0].set_title('raw')\n",