diff --git a/.gitignore b/.gitignore index d1c67878f..713cae1ca 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,4 @@ latexmk-out svg-inkscape *.html +test_graph_visitor*.dot \ No newline at end of file diff --git a/apps/apps/wcsonnet.cxx b/apps/apps/wcsonnet.cxx index 76ac70e82..c1392ceda 100644 --- a/apps/apps/wcsonnet.cxx +++ b/apps/apps/wcsonnet.cxx @@ -33,9 +33,11 @@ int main(int argc, char** argv) { CLI::App app{"wcsonnet is a Wire-Cell Toolkit aware Jsonnet compiler"}; - std::string filename; + std::string filename, output="/dev/stdout"; std::vector load_path, extvars, extcode, tlavars, tlacode; + app.add_option("-o,--output", output, + "Output file"); app.add_option("-P,--path", load_path, "Search paths to consider in addition to those in WIRECELL_PATH")->type_size(1)->allow_extra_args(false); app.add_option("-V,--ext-str", extvars, @@ -86,7 +88,8 @@ int main(int argc, char** argv) m_tlavars, m_tlacode); auto jdat = parser.load(filename); - std::cout << jdat << std::endl; + std::ofstream out(output); + out << jdat << std::endl; return 0; } diff --git a/cfg/layers/mids/uboone/api/img.jsonnet b/cfg/layers/mids/uboone/api/img.jsonnet index 65d6f9b0c..9357f1d15 100644 --- a/cfg/layers/mids/uboone/api/img.jsonnet +++ b/cfg/layers/mids/uboone/api/img.jsonnet @@ -19,7 +19,8 @@ function(services, params) function(anode) local img = low.img(anode); - //-------------- haiwangs +/* + //-------------- haiwang's local img = import "img.jsonnet"; @@ -191,3 +192,6 @@ function(slicing_strategy = "single") high.main(graph, "TbbFlow") +*/ + + diff --git a/cfg/pgraph.jsonnet b/cfg/pgraph.jsonnet index 2ef687c61..2fa9a4575 100644 --- a/cfg/pgraph.jsonnet +++ b/cfg/pgraph.jsonnet @@ -97,7 +97,8 @@ local wc = import "wirecell.jsonnet"; // See intern() for general purpose aggregation of a subgraph. pnode(inode, nin=0, nout=0, uses=[], name=null):: { type: "Pnode", - name: $.prune_array([name, inode.name, ""])[0], + name: $.prune_array([name, wc.cname(inode), ""])[0], + inode: inode, // compound pnodes will not have this attribute edges: [], uses: uses + [inode], iports: [$.port(inode, n) for n in std.range(0,nin)][:nin], diff --git a/cfg/pgrapher/common/sim/nodes.jsonnet b/cfg/pgrapher/common/sim/nodes.jsonnet index 15d1c641d..16eca63cc 100644 --- a/cfg/pgrapher/common/sim/nodes.jsonnet +++ b/cfg/pgrapher/common/sim/nodes.jsonnet @@ -28,6 +28,21 @@ function(params, tools) }, }, nin=1, nout=1, uses=[tools.random]), + // Wrap an IDepoFilter in a IDepoSetFilter. Despite using the + // "DepoSetDrifter" as the wrapper, it can accept any IDepoFilter + // aka IDrifter. This wrapping simply runs many calls to the + // IDepoFilter over the depos in a set. Motivation for doing this + // is to work around the slowdown pathology in Pgrapher when many + // depos are passed in the begining of a large graph. + deposet_filter :: function(depofilt, name="") + g.pnode({ + type: 'DepoSetDrifter', + name: name, + data: { + drifter: wc.tn(depofilt), + }, + }, nin=1, nout=1, uses=[depofilt]), + // Implement "fixed" depo mode like LArG4 uses make_bagger :: function(name="bagger") g.pnode({ type:'DepoBagger', @@ -221,8 +236,9 @@ function(params, tools) misconfigure:: function(params, chndbobj=null) { local split = g.pnode({ - type: "FrameSplitter", - name: "misconsplit" + type: "FrameFanout", + name: "misconsplit", + data: { multiplicity: 2 }, }, nin=1, nout=2), local chsel_static = g.pnode({ diff --git a/cfg/pgrapher/experiment/uboone/sim.jsonnet b/cfg/pgrapher/experiment/uboone/sim.jsonnet index 1a46178c8..950f2986a 100644 --- a/cfg/pgrapher/experiment/uboone/sim.jsonnet +++ b/cfg/pgrapher/experiment/uboone/sim.jsonnet @@ -12,7 +12,21 @@ function(params, tools) local sr = params.shorted_regions, + // A depo set oriented subgraph for WireBoundedDepos. + local wbdepos_sets = [ + sim.deposet_filter( + sim.make_wbdepo(tools.anode, sr.uv + sr.vy, "reject", "nominal"), + "nominal"), + sim.deposet_filter( + sim.make_wbdepo(tools.anode, sr.uv, "accept", "shorteduv"), + "shoteduv"), + sim.deposet_filter( + sim.make_wbdepo(tools.anode, sr.vy, "accept", "shortedvy"), + "shortedvy"), + ], + // The older per-depo WireBoundedDepos subgraph, best to use the + // depo set version above local wbdepos = [ sim.make_wbdepo(tools.anode, sr.uv + sr.vy, "reject", "nominal"), sim.make_wbdepo(tools.anode, sr.uv, "accept", "shorteduv"), @@ -29,6 +43,8 @@ function(params, tools) local pipelines = [g.pipeline([wbdepos[n], baggers[n], depos2frames[n]], "ubsigpipe%d"%n) for n in [0,1,2]], + local pipelines_sets = [g.pipeline([wbdepos_sets[n], depos2frames[n]], "ubsigpipe%d"%n) + for n in [0,1,2]], local ubsigtags = ['ubsig%d'%n for n in [0,1,2]], @@ -36,6 +52,8 @@ function(params, tools) local signal = g.pipeline([f.fanpipe('DepoFanout', pipelines, 'FrameFanin', 'ubsigraph', ubsigtags), sim.make_reframer('ubsigrf', tools.anode, ubsigtags)], 'ubsignal'), + local signal_sets = g.pipeline([f.fanpipe('DepoSetFanout', pipelines_sets, 'FrameFanin', 'ubsigraph', ubsigtags), + sim.make_reframer('ubsigrf', tools.anode, ubsigtags)], 'ubsignal'), // // Noise: @@ -107,6 +125,7 @@ function(params, tools) ret: { signal : signal, + signal_sets : signal_sets, empty_csdb : empty_csdb, miscfg_csdb : miscfg_csdb, make_noise_model :: make_noise_model, diff --git a/cfg/pgrapher/experiment/uboone/sp.jsonnet b/cfg/pgrapher/experiment/uboone/sp.jsonnet index 697289547..d973aaf2b 100644 --- a/cfg/pgrapher/experiment/uboone/sp.jsonnet +++ b/cfg/pgrapher/experiment/uboone/sp.jsonnet @@ -53,13 +53,17 @@ function(params, tools) { // graph is needed to route everything properly. local rawsplit = g.pnode({ - type: "FrameSplitter", - name: "rawsplitter" + // type: "FrameSplitter", + type: "FrameFanout", + name: "rawsplitter", + data: { multiplicity: 2} }, nin=1, nout=2), local sigsplit = g.pnode({ - type: "FrameSplitter", - name: "sigsplitter" + //type: "FrameSplitter", + type: "FrameFanout", + name: "sigsplitter", + data: { multiplicity: 2} }, nin=1, nout=2), local chsel = g.pnode({ @@ -146,18 +150,20 @@ function(params, tools) { } }, nin=2, nout=1), - return: g.intern([rawsplit], [l1merge], [sigproc, sigsplit, chsel, l1spfilter, rawsigmerge, l1merge], - edges=[ - g.edge(rawsplit, sigproc), - g.edge(sigproc, sigsplit), - g.edge(sigsplit, rawsigmerge), - g.edge(sigsplit, l1merge, 1, 1), - - g.edge(rawsplit, rawsigmerge, 1, 1), - g.edge(rawsigmerge, chsel), - g.edge(chsel, l1spfilter), - g.edge(l1spfilter, l1merge), - ], - name="L1SP"), + return: g.intern(innodes=[rawsplit], + outnodes=[l1merge], + centernodes=[sigproc, sigsplit, chsel, l1spfilter, rawsigmerge, l1merge], + edges=[ + g.edge(rawsplit, sigproc), + g.edge(sigproc, sigsplit), + g.edge(sigsplit, rawsigmerge), + g.edge(sigsplit, l1merge, 1, 1), + + g.edge(rawsplit, rawsigmerge, 1, 1), + g.edge(rawsigmerge, chsel), + g.edge(chsel, l1spfilter), + g.edge(l1spfilter, l1merge), + ], + name="L1SP"), }.return diff --git a/cfg/test/test-wbds.jsonnet b/cfg/test/test-wbds.jsonnet new file mode 100644 index 000000000..2d468e105 --- /dev/null +++ b/cfg/test/test-wbds.jsonnet @@ -0,0 +1,25 @@ +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; +local params = import "pgrapher/experiment/uboone/simparams.jsonnet"; +local tools_maker = import 'pgrapher/common/tools.jsonnet'; +local tools = tools_maker(params); +local sim_maker = import "pgrapher/experiment/uboone/sim.jsonnet"; +local sim = sim_maker(params, tools); +local drifter = sim.deposet_filter(sim.drifter, "drifter"); +local graph = pg.pipeline([drifter, sim.signal_sets]); +local app = { + type: 'Pgrapher', + name: "", + data: { + edges: pg.edges(graph), + }, +}; + +local tests = [ + std.assertEqual(wc.tn(drifter), "DepoSetDrifter:drifter"), + std.assertEqual(wc.tn(sim.drifter), "Drifter"), +]; + +pg.uses(graph) + [app] + + diff --git a/cfg/wirecell.jsonnet b/cfg/wirecell.jsonnet index 2a71d58b3..eeaadc185 100644 --- a/cfg/wirecell.jsonnet +++ b/cfg/wirecell.jsonnet @@ -307,6 +307,14 @@ /// example usage: TrackDepos :: self.Component + { type: "TrackDepos" }, + /// Construct a basic configuration node (aka "inode") + cfg(type, name="", data={}) :: { + type:type, name:name, data:data + }, + + /// Return cfg object name or "" + cname(cfgobj) :: if std.objectHas(cfgobj, "name") then cfgobj.name else "", + /// Return canonical "type:name" or just "type" if no name from a /// configuration object. Use this instead of bare names to /// better guard against typos and changes in dependent @@ -320,9 +328,11 @@ /// /// This function can also be applied to objects which happen to /// be produced by pgraph.pnode() - tn(obj) :: if std.objectHas(obj, "name") && obj.name != "" - then obj.type + ":" + obj.name - else obj.type, + tn(cfgobj) :: if std.objectHas(cfgobj, "inode") then + $.tn(cfgobj.inode) + else if self.cname(cfgobj) == "" then + cfgobj.type + else cfgobj.type + ":" + cfgobj.name, // Return a new list where only the first occurrence of any object is kept. diff --git a/img/inc/WireCellImg/ClusterFanin.h b/img/inc/WireCellImg/ClusterFanin.h new file mode 100644 index 000000000..326d4d6d4 --- /dev/null +++ b/img/inc/WireCellImg/ClusterFanin.h @@ -0,0 +1,58 @@ +#ifndef WIRECELLIMG_CLUSTERFANIN +#define WIRECELLIMG_CLUSTERFANIN + +#include "WireCellIface/IClusterFanin.h" +#include "WireCellIface/IConfigurable.h" +#include "WireCellAux/Logger.h" + +namespace WireCell::Img { + + /** Fan N input clusters to 1 output cluster. + + Each input graph becomes a connected subgraph component of the + output graph. + + The relative vertex descriptive ordering within each input + subgraph is retained on ouput. + + Other than edges modified to match the change in vertex + descriptors, no other input graph information is modified + in its output representation. + + */ + class ClusterFanin : public Aux::Logger, + public IClusterFanin, public IConfigurable { + public: + ClusterFanin(); + virtual ~ClusterFanin(); + + // INode, override because we get multiplicity at run time. + virtual std::vector input_types(); + + // IFanin + virtual bool operator()(const input_vector& inv, output_pointer& out); + + // IConfigurable + virtual void configure(const WireCell::Configuration& cfg); + virtual WireCell::Configuration default_configuration() const; + + private: + + /** Config: multiplicity + + The number of inputs to the fan. */ + size_t m_multiplicity{0}; + + /** Config: ident_source + + The produced ICluster::ident() may be set from the graph + input from the port number given by this configuration. + */ + size_t m_ident_source{0}; + + size_t m_count{0}; + }; + +} + +#endif diff --git a/img/inc/WireCellImg/ClusterFanout.h b/img/inc/WireCellImg/ClusterFanout.h index 8eb409a80..6e7bd8206 100644 --- a/img/inc/WireCellImg/ClusterFanout.h +++ b/img/inc/WireCellImg/ClusterFanout.h @@ -3,7 +3,6 @@ #include "WireCellIface/IClusterFanout.h" #include "WireCellIface/IConfigurable.h" -#include "WireCellUtil/TagRules.h" #include "WireCellAux/Logger.h" namespace WireCell::Img { @@ -30,7 +29,6 @@ namespace WireCell::Img { size_t m_count{0}; bool m_trivial{false}; - tagrules::Context m_ft; }; } diff --git a/img/src/ClusterFanin.cxx b/img/src/ClusterFanin.cxx new file mode 100644 index 000000000..262ff645b --- /dev/null +++ b/img/src/ClusterFanin.cxx @@ -0,0 +1,95 @@ +#include "WireCellImg/ClusterFanin.h" +#include "WireCellUtil/Exceptions.h" +#include "WireCellAux/SimpleCluster.h" +#include "WireCellUtil/NamedFactory.h" +WIRECELL_FACTORY(ClusterFanin, WireCell::Img::ClusterFanin, + WireCell::INamed, + WireCell::IClusterFanin, WireCell::IConfigurable) + +using namespace WireCell; + +using WireCell::Aux::SimpleCluster; + +Img::ClusterFanin::ClusterFanin() + : Aux::Logger("ClusterFanin", "glue") +{ +} +Img::ClusterFanin::~ClusterFanin() +{ +} + + +// IConfigurable +WireCell::Configuration Img::ClusterFanin::default_configuration() const +{ + Configuration cfg; + cfg["multiplicity"] = (int) m_multiplicity; + cfg["ident_source"] = (int) m_ident_source; + return cfg; +} + +void Img::ClusterFanin::configure(const WireCell::Configuration& cfg) +{ + const int mult = get(cfg, "multiplicity", (int) m_multiplicity); + if (mult <= 0) { + log->critical("illegal multiplicity={}", mult); + THROW(ValueError() << errmsg{"multiplicity must be positive"}); + } + m_multiplicity = mult; + + int isrc = get(cfg, "ident_source", (int) m_ident_source); + if (isrc < 0 or isrc >= mult) { + log->critical("given illegal ident_source={} with multiplicity={}", isrc, mult); + THROW(ValueError() << errmsg{"illegal ident source"}); + } + m_ident_source = isrc; +} + +// IFanin +bool Img::ClusterFanin::operator()(const input_vector& invec, output_pointer& out) +{ + out = nullptr; + size_t neos = 0; + for (const auto& fr : invec) { + if (!fr) { + ++neos; + } + } + if (neos) { + log->debug("EOS at call={} with {}", m_count, neos); + ++m_count; + return true; + } + if (invec.size() != m_multiplicity) { + log->critical("input vector size={} my multiplicity={}", invec.size(), m_multiplicity); + THROW(ValueError() << errmsg{"input vector size mismatch"}); + } + + cluster_graph_t res; + for (const auto& ic : invec) { + const auto& og = ic->graph(); + + boost::copy_graph(og, res); // need some converter function? + } + + const int ident = invec[m_ident_source]->ident(); + + out = std::make_shared(res, ident); + + const size_t nnodes = boost::num_vertices(res); + const size_t nedges = boost::num_edges(res); + log->debug("fan {} clusters to ident={} with Nv={}, Ne={} at call={}", + m_multiplicity, ident, nnodes, nedges, m_count); + + ++m_count; + return true; +} + + +// INode, override because we get multiplicity at configure time. +std::vector Img::ClusterFanin::input_types() +{ + const std::string tname = std::string(typeid(input_type).name()); + std::vector ret(m_multiplicity, tname); + return ret; +} diff --git a/img/test/depo-ssi-viz.smake b/img/test/depo-ssi-viz.smake index 05c2c51c8..52c02613d 100644 --- a/img/test/depo-ssi-viz.smake +++ b/img/test/depo-ssi-viz.smake @@ -27,7 +27,7 @@ figdir=f"{docdir}/{{dname}}" event_indices = [0] def depo_file(wc): - return depofiles[wc.dname] + return datadir + "/" + depofiles[wc.dname] rule intern_depos: input: depo_file diff --git a/img/test/depo-ssi-viz.yaml b/img/test/depo-ssi-viz.yaml index d181e41d2..4d5707fc2 100644 --- a/img/test/depo-ssi-viz.yaml +++ b/img/test/depo-ssi-viz.yaml @@ -1,11 +1,13 @@ jobcfg: img/test/depo-ssi-viz.jsonnet depofiles: - muon: test/data/muon-depos.npz + muon: muon-depos.npz point: depo-point.npz - cosmics: ../../data/cosmic-500-1-depos.npz + cosmics: cosmic-500-1-depos.npz +datadir: /home/bv/opt/wire-cell-test-data/pdsp/sim/img outdir: build/img/test/depo-ssi-viz docdir: img/docs/depo-ssi-viz -detector: pdsp +# detector: pdsp +detector: uboone variant: nominal drift_speed: "1.56*mm/us" start_time: "314*us" diff --git a/img/test/tenio.org b/img/test/tenio.org new file mode 100644 index 000000000..a333710c1 --- /dev/null +++ b/img/test/tenio.org @@ -0,0 +1,27 @@ +#+title: The tenio test + +This test runs a sequence of similar ~wire-cell~ jobs to transform one +data tier to another. The data tiers are: + +- depo :: ionization deposition groups + +- adc :: frame of ADC waveforms + +- acp :: "a copy" of ADC, for checking basic I/O +- sig :: signal processed results +- img :: imaging results +- ptc :: point cloud results + +Each stage is in a Jsonnet file named + +#+begin_example +tenio--.jssonet +#+end_example + +All stages can be run together with: + +#+begin_example +snakemake -p -jall -s img/test/tenio.smake all +#+end_example + + diff --git a/img/test/test-uboone-img.bats b/img/test/test-uboone-img.bats new file mode 100644 index 000000000..17c79cf8f --- /dev/null +++ b/img/test/test-uboone-img.bats @@ -0,0 +1,163 @@ +#!/usr/bin/env bats + +# Initial test of generating files for input to possible GNNs for +# charge solving and deghosting tasks. + +# Note: this test may break for a while as we consolidate the testing +# system and test data repo. + +############## +# Fixme: these are temporary replacements for what are provided in +# wct-bats.sh, once we merge check-test we should remove them. +function top () { + dirname $(dirname $(dirname $(realpath $BATS_TEST_FILENAME))) +} +function tname () { + basename "$BATS_TEST_FILENAME" .bats +} +function tdir () { + dirname "$BATS_TEST_FILENAME" +} +function resolve_path () { + local want="$1"; shift + if [[ "$want" =~ ^/.* ]] ; then + echo $want + fi + for pathlst in "$(tdir)" $@ ; do + for maybe in $(echo ${pathlst} | tr ":" "\n") + do + if [ -f "${maybe}/$want" ] ; then + echo "Found: ${maybe}/$want" 1>&3 + echo "${maybe}/$want" + return + else + echo "Not found: ${maybe}/$want" 1>&3 + fi + done + done +} +function saveout_path () { + src="$1" ; shift + tgt="$1" + name="$(tname)" + if [ -z "$tgt" ] ; then + tgt="$(basename $src)" + fi + echo "$(top)/build/output/${name}/${tgt}" +} +function saveout () { + src="$1" ; shift + tgt="$(saveout_path $src $1)" + mkdir -p "$(dirname $tgt)" + cp "$src" "$tgt" +} +function cd_tmp () { + + pwd + if [ -n "$WCTEST_TMPDIR" ] ; then + mkdir -p "$WCTEST_TMPDIR" + cd "$WCTEST_TMPDIR" + return + fi + + local which="${1:-test}" + case $which in + suite) cd "$BATS_SUITE_TMPDIR";; + file) cd "$BATS_FILE_TMPDIR";; + run) cd "$BATS_RUN_TMPDIR";; + *) cd "$BATS_TEST_TMPDIR";; # "test" + esac +} +############## + +bimg="blobs-img.npz" +btru="blobs-tru.npz" + +function setup_file () { + + echo "me= $BATS_TEST_FILENAME" + echo "top=$(top)" + # fixme: replace this with WCTEST_{IN,OUT}PUT to allow for variant + # test. + local depos="$(resolve_path test/data/muon-depos.npz $(top))" + [[ -n "$depos" ]] + [[ -s "$depos" ]] + + local log="wire-cell.log" + local cfg="$(resolve_path test-uboone-img.jsonnet)" + [[ -s "$cfg" ]] # temporary test of internal resolve_path + + cd_tmp file + + local cfgjson=cfg.json + local cfgpdf=graph.pdf + + local tlas="-A depos=$depos -A outimg=$bimg -A outtru=$btru" + ### anything we should test for these intermediate outputs? + # tlas = "$tlas -A drifted=drifted.npz -A outadc=adc.npz -A outsig=sig.npz" + if [ -f $cfgjson ] ; then + echo "Reusing $cfgjson" + else + run wcsonnet -o $cfgjson $tlas $cfg + echo "$output" + [[ "$status" -eq 0 ]] + fi + [[ -s $cfgjson ]] + + if [ -f $cfgpdf ] ; then + echo "Reusing $cfgpdf" + else + run wirecell-pgraph dotify $cfgjson $cfgpdf + echo "$output" + [[ "$status" -eq 0 ]] + fi + [[ -s $cfgpdf ]] + saveout $cfgpdf + + if [ -f $log ] ; then + echo "Reusing $log" + else + cmd="wire-cell -l $log -L debug -c $cfgjson" + echo $cmd + run $cmd + echo "$output" + [[ "$status" -eq 0 ]] + fi + saveout $log + # saveout blobs-img.npz + # saveout blobs-tru.npz + [[ -s $bimg ]] + [[ -s $btru ]] +} + +@test "same blobs in zip" { + cd_tmp file + + run bash -c "unzip -v $bimg | grep cluster_ | awk '{print $8}'" + echo "$output" + [[ "$status" -eq 0 ]] + local limg="$output" + + run bash -c "$unzip -v $btru | grep cluster_ | awk '{print $8}'" + echo "$output" + [[ "$status" -eq 0 ]] + local ltru="$output" + + [[ "$(limg)" = "$(ltru)" ]] +} + +@test "same blobs in npz" { + cd_tmp file + + run wirecell-util ls $bimg + echo "$output" + [[ "$status" -eq 0 ]] + local limg="$output" + + run wirecell-util ls $btru + echo "$output" + [[ "$status" -eq 0 ]] + local ltru="$output" + + [[ "$(limg)" = "$(ltru)" ]] +} diff --git a/img/test/test-uboone-img.jsonnet b/img/test/test-uboone-img.jsonnet new file mode 100644 index 000000000..47e532af9 --- /dev/null +++ b/img/test/test-uboone-img.jsonnet @@ -0,0 +1,289 @@ +// This runs uboone sim+sigproc+img with a side pipeline of +// BlobDepoFill to produce blobs with "true" charge based on depos. +// +// Use like: +// wire-cell -A depos=depos.npz -A outimg=blobs-img.npz -A outtru=blobs-tru.npz -c test-uboone-img.jsonnet + + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; +local params = import "pgrapher/experiment/uboone/simparams.jsonnet"; +local tools_maker = import 'pgrapher/common/tools.jsonnet'; +local tools = tools_maker(params); +local sim_maker = import "pgrapher/experiment/uboone/sim.jsonnet"; +local nf_maker = import "pgrapher/experiment/uboone/nf.jsonnet"; +local chndb_maker = import "pgrapher/experiment/uboone/chndb.jsonnet"; +local sp_maker = import "pgrapher/experiment/uboone/sp.jsonnet"; + + +local anodes = tools.anodes; +local anode = anodes[0]; +local aname = anode.data.ident; + +// CHECK THIS FOR MICROBOONE. ProtoDUNE-SP sim needs 314*wc.us. +local time_offset = 314*wc.us; + +local make_random(seeds=[0,1,2,3,4], generator="default") = { + type: "Random", + name: generator + "-" + + std.join('-', std.map(std.toString,seeds)), + data: { + generator: generator, + seeds: seeds, + } +}; + +// don't fluctuate to help keep down variations between tests +local make_drifter(random, xregions, lar, fluctuate=false, name="") = + local d = pg.pnode({ + type: 'Drifter', + name: name, + data: lar { + rng: wc.tn(random), + xregions: xregions, + fluctuate: fluctuate, + }, + }, nin=1, nout=1, uses=[random]); + pg.pnode({ + type: 'DepoSetDrifter', + name: name, + data: { drifter: wc.tn(d) } + }, nin=1, nout=1, uses=[d]); + +local random = make_random(); +local xregions = wc.unique_list(std.flattenArrays([v.faces for v in params.det.volumes])); +local drifter = make_drifter(random, xregions, params.lar); + +local waveform_map = { + type: 'WaveformMap', + name: "", + data: { + filename: "microboone-charge-error.json.bz2", + }, + uses: [], +}; + +local charge_err = pg.pnode({ + type: 'ChargeErrorFrameEstimator', + name: "", + data: { + intag: "gauss", + outtag: 'gauss_error', + anode: wc.tn(anode), + rebin: 4, // this number should be consistent with the waveform_map choice + fudge_factors: [2.31, 2.31, 1.1], // fudge factors for each plane [0,1,2] + time_limits: [12, 800], // the unit of this is in ticks + errors: wc.tn(waveform_map), + }, +}, nin=1, nout=1, uses=[waveform_map, anode]); + + +local slicing(tag="", span=4, active_planes=[0,1,2], masked_planes=[], dummy_planes=[]) = + pg.pnode({ + type: "MaskSlices", + name: "", + data: { + tag: tag, + tick_span: span, + anode: wc.tn(anode), + min_tbin: 0, + max_tbin: 9592, + active_planes: active_planes, + masked_planes: masked_planes, + dummy_planes: dummy_planes, + }, + }, nin=1, nout=1, uses=[anode]); + + +local tiling() = pg.pnode({ + type: "GridTiling", + name: "", + data: { + anode: wc.tn(anode), + face: 0, + } +}, nin=1, nout=1, uses=[anode]); + +local solving(spans=1.0, threshold=0.0) = + pg.pipeline([ + + pg.pnode({ + type: "BlobClustering", + name: "", + data: { spans : spans } + }, nin=1, nout=1), + + pg.pnode({ + type: "BlobGrouping", + name: "" + }, nin=1, nout=1), + + pg.pnode({ + type: "ChargeSolving", + name: "", + data: { + weighting_strategies: ["uniform"], //"uniform", "simple" + } + }, nin=1, nout=1) + ], name=""); + + +local blob_sink(outname, fmt="json") = pg.pnode({ + type: "ClusterFileSink", + name: outname, + data: { + outname: outname, + format: fmt, + } +}, nin=1, nout=0); + + + +// Catch clusters for depo filling +local make_catcher(outtru) = + local bsf = pg.pnode({ + type:'ClusterFanout', + name: "", + data: { + multiplicity: 2 + }}, nin=1, nout=2); + local dbf = pg.pnode({ + type:'BlobDepoFill', + name:"", + data: { + speed: params.lar.drift_speed, // 1.56*wc.mm/wc.us, + time_offset: time_offset, + }}, nin=2, nout=1); + local cfs = blob_sink(outtru); + pg.intern([bsf, dbf], centernodes=[cfs], + edges=[pg.edge(bsf,dbf,1,0), + pg.edge(dbf,cfs,0,0)], + iports=[bsf.iports[0], dbf.iports[1]], + oports=[bsf.oports[0]], + name=""); + +local sim = sim_maker(params, tools); +local chndb = chndb_maker(params, tools).wct("perfect"); + +local depo_file_source(filename, scale=1.0) = + pg.pnode({ + type: 'DepoFileSource', + name: filename, + data: { inname: filename, scale: scale } + }, nin=0, nout=1); + +local make_depos(depos="depos.npz") = + local ds = depo_file_source(depos); + local dsfan = pg.pnode({ + type:'DepoSetFanout', + name:'', + data: { + multiplicity: 2, + }}, nin=1, nout=2); + pg.intern(centernodes=[ds,dsfan], + edges=[pg.edge(ds,dsfan,0,0)], + oports=dsfan.oports, name="deposource"); + + +local sp = sp_maker(params, tools); + +local frametaps(out, tags=[], digitize=false) = + if std.type(out) == "null" || std.length(out) == 0 then + [] + else + local sink = pg.pnode({ + type: 'FrameFileSink', + name: out, + data: { + outname: out, + tags: tags, + digitize: digitize, + }, + }, nin=1, nout=0); + local fan = pg.pnode({ + type: 'FrameFanout', + name: out, + data: { multiplicity: 2 }, + }, nin=1, nout=2); + [pg.intern(innodes=[fan], outnodes=[fan], + centernodes = [sink], + edges=[pg.edge(fan, sink, 1, 0)], + name=out)]; +local depotaps(out) = + if std.type(out) == "null" || std.length(out) == 0 then + [] + else + local sink = pg.pnode({ + type: 'DepoFileSink', + name: out, + data: { outname: out, }, + }, nin=1, nout=0); + local fan = pg.pnode({ + type: 'DepoFanout', + name: out, + data: { multiplicity: 2 }, + }, nin=1, nout=2); + [pg.intern(innodes=[fan], outnodes=[fan], + centernodes = [sink], + edges=[pg.edge(fan, sink, 1, 0)], + name=out)]; + + +// Top level function with TLAs +local make_graph(depos="depos.npz", + outimg="blobs-img.npz", outtru="blobs-tru.npz", + drifted="", outadc="", outsig="") = + local ds = make_depos(depos); + local catcher = make_catcher(outtru); + + local pipeline = [ + ds, + ] + depotaps(drifted) + [ + sim.signal_sets, + sim.add_noise(sim.make_noise_model(anode, sim.miscfg_csdb)), + sim.digitizer(anode, tag="orig") + ] + frametaps(outadc, digitize=true) + [ + nf_maker(params, tools, chndb), + sp, + ] + frametaps(outsig, digitize=false) + [ + charge_err, + slicing("gauss", 109), + tiling(), + solving(), + blob_sink(outimg) + ]; + + local pl = pg.pipeline(pipeline, name='mainpipe'); + // Insert the catcher node. + local pl2 = pg.insert_node(pl, { + tail: {node: 'BlobGrouping', port: 0}, + head: {node: 'ChargeSolving', port: 0} + }, catcher, catcher, 0, 0, name='insert_catcher'); + // Connect the ports 1 of depos and catcher + local graph = pg.intern(centernodes=[pl2], + edges=[pg.edge(ds, catcher, 1, 1)], + name="graph"); + local engine = 'TbbFlow'; + local app = { + // type: 'Pgrapher', + type: engine, + name: "", + data: { + edges: pg.edges(graph), + }, + }; + local cmdline = { + type: "wire-cell", + name: "", + data: { + plugins: [ + "WireCellPgraph", "WireCellTbb", "WireCellSio", + "WireCellGen", "WireCellSigProc", "WireCellImg", + ], + apps: [wc.tn(app)] + } + }; + [cmdline] + pg.uses(graph) + [app]; + + +make_graph diff --git a/img/test/test_clusterfans.cxx b/img/test/test_clusterfans.cxx new file mode 100644 index 000000000..c952e189e --- /dev/null +++ b/img/test/test_clusterfans.cxx @@ -0,0 +1,89 @@ +#include "WireCellImg/ClusterFanin.h" +#include "WireCellImg/ClusterFanout.h" +#include "WireCellAux/SimpleCluster.h" +#include "WireCellUtil/GraphTools.h" +#include "WireCellUtil/Testing.h" + +using namespace WireCell; +using namespace WireCell::Img; +using namespace WireCell::Aux; +using WireCell::GraphTools::mir; + +template +void fan_cfg(Fan& fan, int multiplicity) +{ + auto cfg = fan.default_configuration(); + cfg["multiplicity"] = multiplicity; + fan.configure(cfg); +} + +template +void dump(const Graph& graph, const std::string& name="") +{ + std::cerr << name << " vertices:"; + for (auto vtx : mir(boost::vertices(graph))) { + std::cerr << " " << vtx; + } + std::cerr << "\n" << name << " edges:"; + for (auto edge : mir(boost::edges(graph))) { + auto tvtx = boost::source(edge, graph); + auto hvtx = boost::target(edge, graph); + std::cerr << " (" << tvtx << "," << hvtx << ")"; + } + std::cerr << "\n"; +} + +int main() +{ + const int multiplicity = 2; + const int ident = 42; + + ClusterFanout cfo; + ClusterFanin cfi; + fan_cfg(cfo, multiplicity); + fan_cfg(cfi, multiplicity); + + cluster_graph_t gorig; + size_t num_vertices=0, num_edges=0; + auto v1 = boost::add_vertex(gorig); + ++num_vertices; + auto v2 = boost::add_vertex(gorig); + ++num_vertices; + boost::add_edge(v1, v2, gorig); + ++num_edges; + + dump(gorig, "gorig"); + + auto corig = std::make_shared(gorig, ident); + + // Fanout cluster + ICluster::vector cmany; + Assert(cfo(corig, cmany)); + Assert(cmany.size() == multiplicity); + for (size_t ind=0; ind < (size_t)multiplicity; ++ind) { + Assert(cmany[ind]->ident() == ident); + } + + // Fanin cluster + ICluster::pointer cout = nullptr; + Assert(cfi(cmany, cout)); + Assert(cout); + Assert(cout->ident() == ident); + const auto& gout = cout->graph(); + dump(gout, "gout"); + Assert(boost::num_vertices(gout) == multiplicity*num_vertices); + Assert(boost::num_edges(gout) == multiplicity*num_edges); + + // Fanout EOF + Assert(cfo(nullptr, cmany)); + Assert(cmany.size() == multiplicity); + for (size_t ind=0; ind < (size_t)multiplicity; ++ind) { + Assert(cmany[ind] == nullptr); + } + + // Fanin EOF + Assert(cfi(cmany, cout)); + Assert(cout == nullptr); + + return 0; +} diff --git a/img/wscript_build b/img/wscript_build index 42bb74f6a..11c442e55 100644 --- a/img/wscript_build +++ b/img/wscript_build @@ -1,2 +1,2 @@ -bld.smplpkg('WireCellImg', use='WireCellAux') +bld.smplpkg('WireCellImg', use='WireCellAux', test_use='WireCellImg') diff --git a/pgraph/src/Graph.cxx b/pgraph/src/Graph.cxx index 26d300e9f..f9cf63b0d 100644 --- a/pgraph/src/Graph.cxx +++ b/pgraph/src/Graph.cxx @@ -22,7 +22,11 @@ bool Graph::connect(Node* tail, Node* head, size_t tpind, size_t hpind) Port& tport = tail->output_ports()[tpind]; Port& hport = head->input_ports()[hpind]; if (tport.signature() != hport.signature()) { - l->critical("port signature mismatch: \"{}\" != \"{}\"", tport.signature(), hport.signature()); + l->critical("port signature mismatch:\n\ttail: {}\n\thead: {}\n\ttail data: {}\n\thead data: {}", + tail->ident(), + head->ident(), + tport.signature(), + hport.signature()); THROW(ValueError() << errmsg{"port signature mismatch"}); return false; }