merges

Author: ggmarshall

Diff for: workflow/src/legenddataflow/scripts/par/spms/dsp/trigger_threshold.py

@@ -73,6 +73,12 @@ def par_spms_dsp_trg_thr() -> None:
             )
             log.warning(msg)
 
+        elif len(data) < settings.n_events:
+            msg = (
+                f"number of waveforms '{args.raw_table_name}/waveform_bit_drop' < {settings.n_events}"
+                "in {args.raw_file}, can't build histogram"
+            )
+            raise RuntimeError(msg)
         else:
             # run the DSP with the provided configuration
             log.debug("running the DSP chain")
@@ -87,13 +93,14 @@ def par_spms_dsp_trg_thr() -> None:
             # determine a cutoff for the histogram used to extract the FWHM
             low_cutoff, high_cutoff = np.quantile(wf_current, [0.005, 0.995])
 
-            # make histogram of the curr values
-            h = (
-                hist.new.Regular(settings.n_baseline_bins, low_cutoff, high_cutoff)
-                .Double()
-                .fill(wf_current)
+            # determine hist edges with Friedmann Diaconis Estimator
+            bin_edges = np.histogram_bin_edges(
+                wf_current, bins="fd", range=(low_cutoff, high_cutoff)
             )
 
+            # make histogram of the curr values
+            h = hist.new.Variable(bin_edges).Double().fill(wf_current)
+
             # determine FWHM
             counts = h.view()
             idx_over_half = np.where(counts >= np.max(counts) / 2)[0]
@@ -102,7 +109,7 @@ def par_spms_dsp_trg_thr() -> None:
             fwhm = edges[idx_over_half[-1]] - edges[idx_over_half[0]]
 
             if fwhm <= 0:
-                msg = "determined FWHM of baseline derivative distribution is zero or negative"
+                msg = f"determined FWHM of baseline derivative distribution is so <= 0: {fwhm:.3f}"
                 raise RuntimeError(msg)
 
     log.debug(f"writing out baseline_curr_fwhm = {fwhm}")

Diff for: workflow/src/legenddataflow/scripts/tier/raw_blind.py

@@ -11,7 +11,6 @@
 """
 
 import argparse
-from pathlib import Path
 
 import numexpr as ne
 import numpy as np
@@ -111,12 +110,6 @@ def build_tier_raw_blind() -> None:
     # gets events that should not be blinded
     tokeep = allind[np.logical_not(np.isin(allind, toblind))]
 
-    # make some temp file to write the output to before renaming it
-    rng = np.random.default_rng()
-    rand_num = f"{rng.integers(0,99999):05d}"
-    temp_output = f"{args.output}.{rand_num}"
-    Path(temp_output).parent.mkdir(parents=True, exist_ok=True)
-
     for channel in all_channels:
         try:
             chnum = int(channel[2::])
@@ -126,7 +119,7 @@ def build_tier_raw_blind() -> None:
             store.write_object(
                 chobj,
                 channel,
-                lh5_file=temp_output,
+                lh5_file=args.output,
                 wo_mode="w",
                 **hdf_settings,
             )
@@ -139,7 +132,7 @@ def build_tier_raw_blind() -> None:
                 chobj,
                 group=channel,
                 name="raw",
-                lh5_file=temp_output,
+                lh5_file=args.output,
                 wo_mode="w",
                 **hdf_settings,
             )
@@ -157,11 +150,7 @@ def build_tier_raw_blind() -> None:
             blinded_chobj,
             group=channel,
             name="raw",
-            lh5_file=temp_output,
+            lh5_file=args.output,
             wo_mode="w",
             **hdf_settings,
         )
-
-    # rename the temp file
-    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
-    Path(temp_output).rename(args.output)

Diff for: workflow/src/legenddataflow/scripts/tier/raw_fcio.py

@@ -1,8 +1,6 @@
 import argparse
-from copy import deepcopy
 from pathlib import Path
 
-import numpy as np
 from daq2lh5 import build_raw
 from dbetto import TextDB
 from dbetto.catalog import Props
@@ -36,24 +34,11 @@ def build_tier_raw_fcio() -> None:
     channel_dict = channel_dict.out_spec
     all_config = Props.read_from(channel_dict.gen_config)
 
-    chmap = (
-        TextDB(args.chan_maps, lazy=True).channelmaps.on(args.timestamp).group("system")
-    )
-
     if "geds_config" in channel_dict:
         raise NotImplementedError()
 
     if "spms_config" in channel_dict:
         spm_config = Props.read_from(channel_dict.spms_config)
-        spm_channels = chmap.spms.map("daq.rawid")
-
-        for rawid, chinfo in spm_channels.items():
-            cfg_block = deepcopy(spm_config["FCEventDecoder"]["__output_table_name__"])
-            cfg_block["key_list"] = [chinfo.daq.fc_channel]
-            spm_config["FCEventDecoder"][f"ch{rawid:07d}/raw"] = cfg_block
-
-        spm_config["FCEventDecoder"].pop("__output_table_name__")
-
         Props.add_to(all_config, spm_config)
 
     if "auxs_config" in channel_dict:
@@ -62,17 +47,10 @@ def build_tier_raw_fcio() -> None:
     if "muon_config" in channel_dict:
         raise NotImplementedError()
 
-    rng = np.random.default_rng()
-    rand_num = f"{rng.integers(0,99999):05d}"
-    temp_output = f"{args.output}.{rand_num}"
-
     build_raw(
         args.input,
-        instream_type="FlashCam",
         out_spec=all_config,
-        filekey=temp_output,
+        instream_type="FlashCam",
+        filekey=args.output,
         **settings,
     )
-
-    # rename the temp file
-    Path(temp_output).rename(args.output)

Diff for: workflow/src/legenddataflow/scripts/tier/raw_orca.py

@@ -2,7 +2,6 @@
 import logging
 from pathlib import Path
 
-import numpy as np
 from daq2lh5 import build_raw
 from dbetto import TextDB
 from dbetto.catalog import Props
@@ -101,17 +100,10 @@ def build_tier_raw_orca() -> None:
         )
         Props.add_to(all_config, aux_config)
 
-    rng = np.random.default_rng()
-    rand_num = f"{rng.integers(0,99999):05d}"
-    temp_output = f"{args.output}.{rand_num}"
-
     build_raw(
         args.input,
-        in_stream_type="ORCA",
         out_spec=all_config,
-        filekey=temp_output,
+        in_stream_type="ORCA",
+        filekey=args.output,
         **settings,
     )
-
-    # rename the temp file
-    Path(temp_output).rename(args.output)

Diff for: workflow/src/legenddataflow/scripts/tier/tcm.py

@@ -1,8 +1,6 @@
 import argparse
-from pathlib import Path
 
 import lgdo.lh5 as lh5
-import numpy as np
 from daq2lh5.orca import orca_flashcam
 from dbetto import TextDB
 from dbetto.catalog import Props
@@ -28,10 +26,6 @@ def build_tier_tcm() -> None:
 
     settings = Props.read_from(config_dict["inputs"]["config"])
 
-    rng = np.random.default_rng()
-    temp_output = f"{args.output}.{rng.integers(0, 99999):05d}"
-    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
-
     # get the list of channels by fcid
     ch_list = lh5.ls(args.input, "/ch*")
     fcid_channels = {}
@@ -46,10 +40,8 @@ def build_tier_tcm() -> None:
     for fcid, fcid_dict in fcid_channels.items():
         build_tcm(
             [(args.input, fcid_dict)],
-            out_file=temp_output,
+            out_file=args.output,
             out_name=f"hardware_tcm_{fcid}",
             wo_mode="o",
             **settings,
         )
-
-    Path(temp_output).rename(args.output)