Merge in apply-pointcloud branch with ported clustering.

.gitignore

@@ -80,4 +80,7 @@ configure
 styles
 *.heap
 *.prof
+*.debug
+*.svg
+AutoDict_*
 spng

apps/inc/WireCellApps/Main.h

@@ -81,6 +81,8 @@ namespace WireCell {
         /// Add a log sink, reserved names 'stdout' and 'stderr' or a filename.
         void add_logsink(const std::string& log, const std::string& level = "");
 
+        void apply_log_config();
+
         /// Set a minimum level to emit a message for a given
         /// log. (levels: critical, error, warn, info, debug, trace).
         void set_loglevel(const std::string& log, const std::string& level = "");
@@ -107,6 +109,9 @@ namespace WireCell {
         // Limit number of threads.  0 means set no limit.  This is
         // only relevant if we are built with TBB support
         int m_threads{0};
+
+        std::unordered_map<std::string, std::string> m_log_levels;
+
     };
 
 }  // namespace WireCell

apps/src/Main.cxx

@@ -226,8 +226,14 @@ void Main::add_logsink(const std::string& log, const std::string& level)
 }
 void Main::set_loglevel(const std::string& log, const std::string& level)
 {
-    // std::cerr << "Set log \""<<log<<"\" to level \"" << level << "\"\n";
-    Log::set_level(level, log);
+    m_log_levels[log] = level;
+    // Log::set_level(level, log);
+}
+void Main::apply_log_config()
+{
+    for (const auto& [log,level] : m_log_levels) {
+        Log::set_level(level, log);
+    }
 }
 void Main::add_config(const std::string& filename) { m_cfgfiles.push_back(filename); }
 
@@ -251,8 +257,7 @@ void Main::initialize()
     Log::set_pattern("[%H:%M:%S.%03e] %L [%^%=8n%$] %v");
     log = Log::logger("main");
     log->set_pattern("[%H:%M:%S.%03e] %L [  main  ] %v");
-    set_loglevel("", "debug");
-    // log->debug("logging to \"main\"");
+    log->debug("logging to \"main\"");
 
     // Load configuration files
     for (auto filename : m_cfgfiles) {
@@ -328,6 +333,10 @@ void Main::initialize()
         namobj->set_name(name);
     }
 
+    // We do this after setting names as INamed may have created a logger.
+    apply_log_config();
+    Log::fill_levels();
+
     // Finally, ask any configurables for their default, merge with
     // user config and give back.
     for (auto c : m_cfgmgr.all()) {

aux/docs/ClusterArrays.org

@@ -207,3 +207,9 @@ The implementation landscape for persisting ~ICluster~ is illustrated:
 
 Solid thick black edges represent exchange of data between WCT data flow graph nodes.  Solid thin lines represent code dependency.  Thick dashed edges represent file I/O.  Note ~ClusterFile{Source,Sink}~ directly convert ~ICluster~ while ~TensorFile{Source,Sink}~ are general and the conversion to ~ITensor~ is performed by ~TensorCluster~ and ~ClusterTensor~.
 
+* Meta :noexport:
+
+
+#+begin_src sh :results none :output none
+scp -r ClusterArrays.html ClusterArrays.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/docs/cluster-shadow.org

@@ -63,3 +63,10 @@ each edge endpoint and then perform pair-wise set-theoretic operations
 on the ~Rectangles~ object from each.
 
 Examples and/or pointers to tests t.b.d.
+
+* Meta :noexport:
+
+
+#+begin_src sh :results none :output none
+scp -r cluster-shadow.html cluster-shadow.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/docs/frame-files.org

@@ -242,3 +242,9 @@ additional metadata:
 - ~tag="tag"~ :: the associated index trace tag.
 Note, it is undefined behavior if no matching index tensor exists.
 
+* Meta :noexport:
+
+
+#+begin_src sh :results none :output none
+scp -r frame-files.html frame-files.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/docs/frame-tensor.org

@@ -192,3 +192,9 @@ additional metadata:
 
 Note, it is undefined behavior if no matching index tensor exists.
 
+* Meta :noexport:
+
+
+#+begin_src sh :results none :output none
+scp -r frame-tensor.html frame-tensor.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/docs/noise.org

@@ -1,5 +1,6 @@
 #+title: WCT Noise Modeling
 #+include: ../../docs/include-topic.org
+#+export_file_name: noise-aux
 
 * Overview
 
@@ -209,3 +210,9 @@ Or, run it all together as:
 #+begin_example
 aux/test/test-noise-roundtrip.sh 
 #+end_example
+
+* Meta :noexport:
+
+#+begin_src sh :results none :output none
+scp -r noise-aux.html noise-aux.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/docs/talks/noise-presentation.org

@@ -803,6 +803,11 @@ Interpolate so $n' \to n'' = \frac{N_2}{2} \triangleq \frac{N''_1}{2R_{21}},\ N_
 If $N''_1 > N_2$ we may alias by pretending same periods.
 
 
+* Meta :noexport:
+
+#+begin_src sh :results none :output none
+scp -r noise-presentation.pdf  hierocles.bnl:public_html/wire-cell/docs/
+#+end_src
 
 
 

aux/docs/tensor-data-model.org

@@ -368,3 +368,8 @@ ITensor::vector as_tensors(<concrete types>);
 <concrete types> as_<concrete name>(<ITensor types>);
 #+end_example
 
+* Meta :noexport:
+
+#+begin_src sh :results none :output none
+scp -r tensor-data-model.html tensor-data-model.pdf hierocles.bnl:public_html/wire-cell/docs/
+#+end_src

aux/inc/WireCellAux/Bee.h

@@ -0,0 +1,23 @@
+// An API to interface with Bee that deals in IData.
+
+#ifndef WIRECELLAUX_BEE
+#define WIRECELLAUX_BEE
+
+#include "WireCellUtil/Bee.h"
+#include "WireCellIface/IBlobSet.h"
+#include "WireCellIface/IBlobSampler.h"
+
+namespace WireCell::Aux::Bee {
+
+    using WireCell::Bee::Points;
+
+    // Convert a blob set to a Bee Points object by applying the sampler.  The
+    // sampler is expected to produce point cloud arrays named: "x", "y", "z"
+    // for point positions and may produce "charge_val" for a charge measure.
+    Points dump(IBlobSet::pointer bs, IBlobSampler::pointer sampler);
+    Points dump(const IBlob::vector& blobs, IBlobSampler::pointer sampler);
+
+}
+
+
+#endif

aux/inc/WireCellAux/BeeTools.h

@@ -0,0 +1,12 @@
+// An API to interface with Bee that deals in IData.
+
+#ifndef WIRECELLAUX_BEETOOLS
+#define WIRECELLAUX_BEETOOLS
+
+namespace WireCell::Aux::Bee {
+
+
+}
+
+
+#endif

aux/inc/WireCellAux/BlobTools.h

@@ -4,7 +4,7 @@
 #ifndef WIRECELLAUX_BLOBTOOLS
 #define WIRECELLAUX_BLOBTOOLS
 
-#include "WireCellIface/IBlobSampler.h"
+#include "WireCellIface/IBlobSet.h"
 
 namespace WireCell::Aux {
 
@@ -54,5 +54,8 @@ namespace WireCell::Aux {
         }
     };
 
+    // Dump a summary of a cluster graph.  If fingerprint, include more verbosity of channel-level info.
+    std::string dumps(const IBlobSet::pointer& bs);
+
 }
 #endif

aux/inc/WireCellAux/ClusterHelpers.h

@@ -22,7 +22,8 @@
 
 namespace WireCell::Aux {
 
-    std::string dumps(const cluster_graph_t& cgraph);
+    // Dump a summary of a cluster graph.  If fingerprint, include more verbosity of channel-level info.
+    std::string dumps(const cluster_graph_t& cgraph, bool fingerprint=false);
 
     /// Return JSON representation of the cluster.
     Json::Value jsonify(const cluster_graph_t& cgraph);

aux/inc/WireCellAux/SimpleBlob.h

@@ -56,6 +56,13 @@ namespace WireCell::Aux {
           , m_blobs(blobs)
         {
         }
+        // Leaky abstraction: a blob set may span multiple slices
+        SimpleBlobSet(int ident, const IBlob::vector& blobs)
+          : m_ident(ident)
+          , m_slice(nullptr)
+          , m_blobs(blobs)
+        {
+        }
         virtual ~SimpleBlobSet();
 
         virtual int ident() const { return m_ident; }
@@ -64,6 +71,8 @@ namespace WireCell::Aux {
 
         virtual IBlob::vector blobs() const { return m_blobs; }
 
+        void insert(const IBlob::pointer& blob) { m_blobs.push_back(blob); }
+
         int m_ident;
         ISlice::pointer m_slice;
         IBlob::vector m_blobs;

aux/inc/WireCellAux/SimpleSlice.h

@@ -14,6 +14,10 @@ namespace WireCell::Aux {
         SimpleSlice(IFrame::pointer frame, int ident, double start, double span, const ISlice::map_t& activity)
             : m_frame(frame), m_ident(ident), m_start(start), m_span(span), m_activity(activity) { }
 
+        // Construct with no activity, user must fill in after construction.
+        SimpleSlice(IFrame::pointer frame, int ident, double start, double span)
+            : m_frame(frame), m_ident(ident), m_start(start), m_span(span) { }
+
         virtual ~SimpleSlice() {}
 
         // ISlice interface
@@ -22,6 +26,7 @@ namespace WireCell::Aux {
         double start() const { return m_start; }
         double span() const { return m_span; }
         map_t activity() const { return m_activity; }
+        map_t& activity() { return m_activity; }
     };
 }
 

aux/inc/WireCellAux/Util.h

@@ -158,7 +158,7 @@ namespace WireCell {
             return ITensor::pointer(st);
         }
 
-    };  // namespace Aux
+    }  // namespace Aux
 }  // namespace WireCell
 
 #endif  // WIRECELLSIG_UTIL

aux/src/Bee.cxx

@@ -0,0 +1,36 @@
+#include "WireCellAux/Bee.h"
+#include "WireCellUtil/RaySamplers.h"
+
+using namespace WireCell;
+
+Aux::Bee::Points Aux::Bee::dump(IBlobSet::pointer bs, IBlobSampler::pointer sampler)
+{
+    return dump(bs->blobs(), sampler);
+}
+
+Aux::Bee::Points Aux::Bee::dump(const IBlob::vector& blobs, IBlobSampler::pointer sampler)
+{
+    Aux::Bee::Points bee;    
+
+    for (const auto& iblob : blobs) {
+        auto [pc, aux] = sampler->sample_blob(iblob, iblob->ident());
+        auto x = pc.get("x")->elements<double>();
+        auto y = pc.get("y")->elements<double>();
+        auto z = pc.get("z")->elements<double>();
+        PointCloud::Array::span_t<double> q;
+        auto qarr = pc.get("charge_val");
+        if (qarr) {
+            q = qarr->elements<double>();
+        }
+
+        const size_t npts = x.size();
+        for (size_t ind=0; ind<npts; ++ind) {
+            double charge = 0;
+            if (q.size()) {
+                charge = q[ind];
+            }
+            bee.append({x[ind], y[ind], z[ind]}, charge);
+        }
+    }
+    return bee;
+}

aux/src/BlobTools.cxx

@@ -1,4 +1,5 @@
 #include "WireCellAux/BlobTools.h"
+#include "WireCellUtil/Units.h"
 
 using namespace WireCell;
 
@@ -33,3 +34,38 @@ Aux::BlobCategory::BlobCategory(const IBlob::pointer& iblob)
     }
     stat = Status::okay;
 }
+
+
+#include <unordered_set>
+std::string Aux::dumps(const IBlobSet::pointer& bs)
+{
+    std::stringstream ss;
+
+    const size_t nbcats = BlobCategory::size();
+    std::vector<size_t> blob_cats(nbcats, 0);
+    std::unordered_set<ISlice::pointer> slices;
+
+    for (auto iblob: bs->blobs()) {
+        Aux::BlobCategory bcat(iblob);
+        ++blob_cats[bcat.num()];
+        slices.insert(iblob->slice());
+    }
+
+    ss << "Blobset ident:" << bs->ident()
+       << " spans "  << slices.size() << " slices, main slice:";
+    auto islice = bs->slice();
+    if (islice) {
+        ss << islice->ident() << " with " << islice->activity().size() << " activities, span="
+           << islice->start()/units::us << "+" << islice->span()/units::us << " us";
+
+    }
+    else {
+        ss << "(none)";
+    }
+    ss << " bcats:[";
+    for (size_t icat=0; icat < nbcats; ++icat) {
+        ss << " " << BlobCategory::to_str(icat) << "=" << blob_cats[icat];
+    }
+    ss << " ]";
+    return ss.str();
+}