Prepare WCT to make GNN training samples

.gitignore

@@ -69,3 +69,4 @@ latexmk-out
 
 svg-inkscape
 *.html
+test_graph_visitor*.dot

apps/apps/wcsonnet.cxx

@@ -33,9 +33,11 @@ int main(int argc, char** argv)
 {
     CLI::App app{"wcsonnet is a Wire-Cell Toolkit aware Jsonnet compiler"};
 
-    std::string filename;
+    std::string filename, output="/dev/stdout";
     std::vector<std::string> load_path, extvars, extcode, tlavars, tlacode;
 
+    app.add_option("-o,--output", output,
+                   "Output file");
     app.add_option("-P,--path", load_path,
                    "Search paths to consider in addition to those in WIRECELL_PATH")->type_size(1)->allow_extra_args(false);
     app.add_option("-V,--ext-str", extvars,
@@ -86,7 +88,8 @@ int main(int argc, char** argv)
                            m_tlavars, m_tlacode);
 
     auto jdat = parser.load(filename);
-    std::cout << jdat << std::endl;
+    std::ofstream out(output);
+    out << jdat << std::endl;
 
     return 0;
 }

cfg/layers/mids/uboone/api/img.jsonnet

@@ -19,7 +19,8 @@ function(services, params) function(anode)
     local img = low.img(anode);
 
 
-    //-------------- haiwangs
+/*
+    //-------------- haiwang's
 
 local img = import "img.jsonnet";
 
@@ -191,3 +192,6 @@ function(slicing_strategy = "single")
 
     high.main(graph, "TbbFlow")
 
+*/
+
+

cfg/pgraph.jsonnet

@@ -97,7 +97,8 @@ local wc = import "wirecell.jsonnet";
     // See intern() for general purpose aggregation of a subgraph.
     pnode(inode, nin=0, nout=0, uses=[], name=null):: {
         type: "Pnode",
-        name: $.prune_array([name, inode.name, ""])[0],
+        name: $.prune_array([name, wc.cname(inode), ""])[0],
+        inode: inode,  // compound pnodes will not have this attribute
         edges: [],
         uses: uses + [inode],
         iports: [$.port(inode, n) for n in std.range(0,nin)][:nin],

cfg/pgrapher/common/sim/nodes.jsonnet

@@ -28,6 +28,21 @@ function(params, tools)
         },
     }, nin=1, nout=1, uses=[tools.random]),
 
+    // Wrap an IDepoFilter in a IDepoSetFilter.  Despite using the
+    // "DepoSetDrifter" as the wrapper, it can accept any IDepoFilter
+    // aka IDrifter.  This wrapping simply runs many calls to the
+    // IDepoFilter over the depos in a set.  Motivation for doing this
+    // is to work around the slowdown pathology in Pgrapher when many
+    // depos are passed in the begining of a large graph.
+    deposet_filter :: function(depofilt, name="")
+        g.pnode({
+            type: 'DepoSetDrifter',
+            name: name,
+            data: {
+                drifter: wc.tn(depofilt),
+            },
+        }, nin=1, nout=1, uses=[depofilt]),
+
     // Implement "fixed" depo mode like LArG4 uses
     make_bagger :: function(name="bagger") g.pnode({
         type:'DepoBagger',
@@ -221,8 +236,9 @@ function(params, tools)
     misconfigure:: function(params, chndbobj=null) {
 
         local split = g.pnode({
-            type: "FrameSplitter",
-            name: "misconsplit"
+            type: "FrameFanout",
+            name: "misconsplit",
+            data: { multiplicity: 2 },
         }, nin=1, nout=2),
 
         local chsel_static = g.pnode({

cfg/pgrapher/experiment/uboone/sim.jsonnet

@@ -12,7 +12,21 @@ function(params, tools)
 
     local sr = params.shorted_regions,
 
+    // A depo set oriented subgraph for WireBoundedDepos.
+    local wbdepos_sets = [
+        sim.deposet_filter(
+            sim.make_wbdepo(tools.anode, sr.uv + sr.vy, "reject", "nominal"),
+            "nominal"),
+        sim.deposet_filter(
+            sim.make_wbdepo(tools.anode, sr.uv, "accept", "shorteduv"),
+            "shoteduv"),
+        sim.deposet_filter(
+            sim.make_wbdepo(tools.anode, sr.vy, "accept", "shortedvy"),
+            "shortedvy"),
+    ],
 
+    // The older per-depo WireBoundedDepos subgraph, best to use the
+    // depo set version above
     local wbdepos = [
         sim.make_wbdepo(tools.anode, sr.uv + sr.vy, "reject", "nominal"),
         sim.make_wbdepo(tools.anode, sr.uv, "accept", "shorteduv"),
@@ -29,13 +43,17 @@ function(params, tools)
 
     local pipelines = [g.pipeline([wbdepos[n], baggers[n], depos2frames[n]], "ubsigpipe%d"%n)
                        for n in [0,1,2]],
+    local pipelines_sets = [g.pipeline([wbdepos_sets[n], depos2frames[n]], "ubsigpipe%d"%n)
+                            for n in [0,1,2]],
 
 
     local ubsigtags = ['ubsig%d'%n for n in [0,1,2]],
 
     local signal = g.pipeline([f.fanpipe('DepoFanout', pipelines, 'FrameFanin', 'ubsigraph', ubsigtags),
                                sim.make_reframer('ubsigrf', tools.anode, ubsigtags)], 'ubsignal'),
 
+    local signal_sets = g.pipeline([f.fanpipe('DepoSetFanout', pipelines_sets, 'FrameFanin', 'ubsigraph', ubsigtags),
+                                    sim.make_reframer('ubsigrf', tools.anode, ubsigtags)], 'ubsignal'),
 
     //
     // Noise:
@@ -107,6 +125,7 @@ function(params, tools)
 
     ret: {
         signal : signal,
+        signal_sets : signal_sets,
         empty_csdb : empty_csdb,
         miscfg_csdb : miscfg_csdb,
         make_noise_model :: make_noise_model,

cfg/pgrapher/experiment/uboone/sp.jsonnet

@@ -53,13 +53,17 @@ function(params, tools) {
     // graph is needed to route everything properly.
 
     local rawsplit = g.pnode({
-        type: "FrameSplitter",
-        name: "rawsplitter"
+        // type: "FrameSplitter",
+        type: "FrameFanout",
+        name: "rawsplitter",
+        data: { multiplicity: 2}
     }, nin=1, nout=2),
 
     local sigsplit = g.pnode({
-        type: "FrameSplitter",
-        name: "sigsplitter"
+        //type: "FrameSplitter",
+        type: "FrameFanout",
+        name: "sigsplitter",
+        data: { multiplicity: 2}
     }, nin=1, nout=2),
 
     local chsel = g.pnode({
@@ -146,18 +150,20 @@ function(params, tools) {
         }
     }, nin=2, nout=1),
 
-    return: g.intern([rawsplit], [l1merge], [sigproc, sigsplit, chsel, l1spfilter, rawsigmerge, l1merge],
-                          edges=[
-                              g.edge(rawsplit, sigproc),
-                              g.edge(sigproc, sigsplit),
-                              g.edge(sigsplit, rawsigmerge),
-                              g.edge(sigsplit, l1merge, 1, 1),
-
-                              g.edge(rawsplit, rawsigmerge, 1, 1),
-                              g.edge(rawsigmerge, chsel),
-                              g.edge(chsel, l1spfilter),
-                              g.edge(l1spfilter, l1merge),
-                          ],
-                          name="L1SP"),
+    return: g.intern(innodes=[rawsplit],
+                     outnodes=[l1merge],
+                     centernodes=[sigproc, sigsplit, chsel, l1spfilter, rawsigmerge, l1merge],
+                     edges=[
+                         g.edge(rawsplit, sigproc),
+                         g.edge(sigproc, sigsplit),
+                         g.edge(sigsplit, rawsigmerge),
+                         g.edge(sigsplit, l1merge, 1, 1),
+
+                         g.edge(rawsplit, rawsigmerge, 1, 1),
+                         g.edge(rawsigmerge, chsel),
+                         g.edge(chsel, l1spfilter),
+                         g.edge(l1spfilter, l1merge),
+                     ],
+                     name="L1SP"),
 
 }.return

cfg/test/test-wbds.jsonnet

@@ -0,0 +1,25 @@
+local wc = import "wirecell.jsonnet";
+local pg = import "pgraph.jsonnet";
+local params = import "pgrapher/experiment/uboone/simparams.jsonnet";
+local tools_maker = import 'pgrapher/common/tools.jsonnet';
+local tools = tools_maker(params);
+local sim_maker = import "pgrapher/experiment/uboone/sim.jsonnet";
+local sim = sim_maker(params, tools);
+local drifter = sim.deposet_filter(sim.drifter, "drifter");
+local graph = pg.pipeline([drifter, sim.signal_sets]);
+local app = {
+    type: 'Pgrapher',
+    name: "",
+    data: {
+        edges: pg.edges(graph),
+    },
+};
+
+local tests = [
+    std.assertEqual(wc.tn(drifter), "DepoSetDrifter:drifter"),
+    std.assertEqual(wc.tn(sim.drifter), "Drifter"),
+];
+
+pg.uses(graph) + [app]
+
+

cfg/wirecell.jsonnet

@@ -307,6 +307,14 @@
     /// example usage: 
     TrackDepos :: self.Component + { type: "TrackDepos" },
 
+    /// Construct a basic configuration node (aka "inode")
+    cfg(type, name="", data={}) :: {
+        type:type, name:name, data:data
+    },
+
+    /// Return cfg object name or ""
+    cname(cfgobj) :: if std.objectHas(cfgobj, "name") then cfgobj.name else "",
+
     /// Return canonical "type:name" or just "type" if no name from a
     /// configuration object.  Use this instead of bare names to
     /// better guard against typos and changes in dependent
@@ -320,9 +328,11 @@
     ///
     /// This function can also be applied to objects which happen to
     /// be produced by pgraph.pnode()
-    tn(obj) :: if std.objectHas(obj, "name") && obj.name != ""
-               then obj.type + ":" + obj.name
-               else obj.type,
+    tn(cfgobj) :: if std.objectHas(cfgobj, "inode") then
+                      $.tn(cfgobj.inode)
+                  else if self.cname(cfgobj) == "" then
+                      cfgobj.type
+                  else cfgobj.type + ":" + cfgobj.name,
 
 
     // Return a new list where only the first occurrence of any object is kept.

img/inc/WireCellImg/ClusterFanin.h

@@ -0,0 +1,58 @@
+#ifndef WIRECELLIMG_CLUSTERFANIN
+#define WIRECELLIMG_CLUSTERFANIN
+
+#include "WireCellIface/IClusterFanin.h"
+#include "WireCellIface/IConfigurable.h"
+#include "WireCellAux/Logger.h"
+
+namespace WireCell::Img {
+
+    /** Fan N input clusters to 1 output cluster.
+
+        Each input graph becomes a connected subgraph component of the
+        output graph.
+
+        The relative vertex descriptive ordering within each input
+        subgraph is retained on ouput.
+
+        Other than edges modified to match the change in vertex
+        descriptors, no other input graph information is modified
+        in its output representation.
+
+     */
+    class ClusterFanin :  public Aux::Logger,
+                          public IClusterFanin, public IConfigurable {
+      public:
+        ClusterFanin();
+        virtual ~ClusterFanin();
+
+        // INode, override because we get multiplicity at run time.
+        virtual std::vector<std::string> input_types();
+
+        // IFanin
+        virtual bool operator()(const input_vector& inv, output_pointer& out);
+
+        // IConfigurable
+        virtual void configure(const WireCell::Configuration& cfg);
+        virtual WireCell::Configuration default_configuration() const;
+
+      private:
+
+        /** Config: multiplicity
+
+            The number of inputs to the fan. */
+        size_t m_multiplicity{0};
+
+        /** Config: ident_source
+
+            The produced ICluster::ident() may be set from the graph
+            input from the port number given by this configuration.
+        */
+        size_t m_ident_source{0};
+
+        size_t m_count{0};
+    };
+
+}
+
+#endif

img/inc/WireCellImg/ClusterFanout.h

@@ -3,7 +3,6 @@
 
 #include "WireCellIface/IClusterFanout.h"
 #include "WireCellIface/IConfigurable.h"
-#include "WireCellUtil/TagRules.h"
 #include "WireCellAux/Logger.h"
 
 namespace WireCell::Img {
@@ -30,7 +29,6 @@ namespace WireCell::Img {
         size_t m_count{0};
 
         bool m_trivial{false};
-        tagrules::Context m_ft;
 
     };
 }