diff --git a/apps/inc/WireCellApps/ConfigSchema.h b/apps/inc/WireCellApps/ConfigSchema.h new file mode 100644 index 000000000..18166d4a2 --- /dev/null +++ b/apps/inc/WireCellApps/ConfigSchema.h @@ -0,0 +1,33 @@ +/** Produce a schema of all registered components. + + Eg + + wire-cell -a ConfigSchema -p WireCellApps -p WireCellGen + + */ + +#ifndef WIRECELLAPPS_CONFIGSCHEMA +#define WIRECELLAPPS_CONFIGSCHEMA + +#include "WireCellIface/IApplication.h" +#include "WireCellIface/IConfigurable.h" +#include "WireCellUtil/Configuration.h" + +namespace WireCellApps { + + class ConfigSchema : public WireCell::IApplication, public WireCell::IConfigurable { + WireCell::Configuration m_cfg; + + public: + ConfigSchema(); + virtual ~ConfigSchema(); + + virtual void execute(); + + virtual void configure(const WireCell::Configuration& config); + virtual WireCell::Configuration default_configuration() const; + }; + +} // namespace WireCellApps + +#endif diff --git a/apps/src/ConfigSchema.cxx b/apps/src/ConfigSchema.cxx new file mode 100644 index 000000000..af95953fc --- /dev/null +++ b/apps/src/ConfigSchema.cxx @@ -0,0 +1,253 @@ +#include "WireCellApps/ConfigSchema.h" + +#include "WireCellIface/INode.h" + +#include "WireCellUtil/String.h" +#include "WireCellUtil/Type.h" +#include "WireCellUtil/Persist.h" +#include "WireCellUtil/NamedFactory.h" +#include "WireCellUtil/ConfigManager.h" +#include "WireCellUtil/Logging.h" +#include "WireCellUtil/Exceptions.h" + + +WIRECELL_FACTORY(ConfigSchema, + WireCellApps::ConfigSchema, + WireCell::IApplication, + WireCell::IConfigurable) + +using spdlog::info; +using spdlog::warn; + +using namespace WireCell; +using namespace WireCellApps; + +ConfigSchema::ConfigSchema() + : m_cfg(default_configuration()) +{ +} + +ConfigSchema::~ConfigSchema() {} + +void ConfigSchema::configure(const Configuration& config) { m_cfg = config; } + +WireCell::Configuration ConfigSchema::default_configuration() const +{ + // yo dawg, I heard you liked dumping so I made a dumper that dumps the dumper. + Configuration cfg; + cfg["filename"] = "/dev/stdout"; + cfg["components"] = Json::arrayValue; + return cfg; +} + +// the "name" is the local type name. We don't really have that available so +// instead if the cfg is a value of an object, the name will be the attribute +// key name. O.w., empty. This is something to fix by hand after the dump. +Configuration dump_schema(const Configuration& cfg, const std::string& name=""); +Configuration dump_object(const Configuration& cfg, const std::string& name=""); +Configuration dump_number(const Configuration& cfg, const std::string& name=""); +Configuration dump_string(const Configuration& cfg, const std::string& name=""); +Configuration dump_array(const Configuration& cfg, const std::string& name=""); +Configuration dump_bool(const Configuration& cfg, const std::string& name=""); +Configuration dump_null(const Configuration& cfg, const std::string& name=""); + +static +void maybe_name(Configuration& schema, const std::string& name) +{ + if (name.empty()) return; + schema["name"] = name; +} + +Configuration dump_object(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "record"; + maybe_name(schema, name); + schema["fields"] = Json::arrayValue; + + for (const auto& key : cfg.getMemberNames()) { + auto val = cfg[key]; + Configuration field; + field["name"] = key; + field["item"] = dump_schema(val, key); + if (! val.empty()) { + field["default"] = val; + } + schema["fields"].append(field); + } + + return schema; +} + +Configuration dump_number(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "number"; + maybe_name(schema, name); + // omit: path, deps. + + // How can we possibly guess these these distinctions? + if(cfg.isInt()) { + schema["dtype"] = "i4"; + } + else { + schema["dtype"] = "f8"; + } + + return schema; +} + +Configuration dump_string(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "string"; + maybe_name(schema, name); + // omit: path, deps, pattern/format. + + return schema; +} + +Configuration dump_array(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "sequence"; + maybe_name(schema, name); + + if (cfg.empty()) { + schema["item"] = "null"; + } + else { + schema["item"] = dump_schema(cfg[0]); + } + return schema; +} + +Configuration dump_bool(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "boolean"; + maybe_name(schema, name); + + return schema; +} + +Configuration dump_null(const Configuration& cfg, const std::string& name) +{ + Configuration schema; + schema["schema"] = "null"; + maybe_name(schema, name); + + return schema; +} + + +Configuration dump_schema(const Configuration& cfg, const std::string& name) +{ + if (cfg.isObject()) { + return dump_object(cfg); + } + if (cfg.isNumeric()) { + return dump_number(cfg); + } + if (cfg.isString()) { + return dump_string(cfg); + } + if (cfg.isArray()) { + return dump_array(cfg); + } + if (cfg.isBool()) { + return dump_bool(cfg); + } + if (cfg.isNull()) { + return dump_null(cfg); + } + raise("unknown type: %s", cfg); + // quell compiler warning about no return value. + return Configuration(); +} + +void ConfigSchema::execute() +{ + // ConfigManager cm; + int nfailed = 0; + + std::vector comps; + for (auto jone : m_cfg["components"]) { + comps.push_back(jone.asString()); + } + + if (comps.empty()) { + comps = Factory::known_types(); + } + + std::unordered_map categories = { + { INode::unknown,"unknown" }, + { INode::sourceNode, "source" }, + { INode::sinkNode, "sink" }, + { INode::functionNode, "function" }, + { INode::queuedoutNode, "queuedout" }, + { INode::joinNode, "join" }, + { INode::splitNode, "split" }, + { INode::faninNode, "fanin" }, + { INode::fanoutNode, "fanout" }, + { INode::multioutNode, "multiout" }, + { INode::hydraNode, "hydra" }, + }; + + Configuration all_schema; + + for (auto const& [wctype, tinfo] : NamedFactoryRegistryBase::known_type_info()) { + Configuration schema; + schema["cppname"] = tinfo.cppname; + + schema["interfaces"] = Json::arrayValue; + for (auto const& intname : tinfo.intnames) { + schema["interfaces"].append(intname); + } + + auto inode = Factory::lookup_tn(wctype, true, true); + if (inode) { + Configuration node; + + node["category"] = categories[inode->category()]; + node["signature"]= demangle( inode->signature() ); + node["concurrency"] = inode->concurrency(); + + { + Configuration ports = Json::arrayValue; + for (const auto& port : inode->input_types()) { + ports.append( demangle(port) ); + } + node["iports"] = ports; + } + { + Configuration ports = Json::arrayValue; + for (const auto& port : inode->output_types()) { + ports.append( demangle(port) ); + } + node["oports"] = ports; + } + schema["node"] = node; + } + all_schema[wctype] = schema; + } + + for (auto c : comps) { + auto [wctype, instname] = String::parse_pair(convert(c)); + + Configuration cfg; + try { + auto cfgobj = Factory::lookup(wctype, instname); + cfg = cfgobj->default_configuration(); + } + catch (FactoryException& fe) { + warn("failed lookup component: \"{}\":\"{}\"", wctype, instname); + ++nfailed; + continue; + } + + all_schema[wctype]["config"] = dump_schema(cfg); + } + + Persist::dump(get(m_cfg, "filename"), all_schema); +} diff --git a/apps/test/add-missing-schema.jsonnet b/apps/test/add-missing-schema.jsonnet new file mode 100644 index 000000000..4772b6317 --- /dev/null +++ b/apps/test/add-missing-schema.jsonnet @@ -0,0 +1,34 @@ +// Add schema for configuration object attributes that the C++ code fail to +// create in the object returned by default_configuration(). +// +// See test-config-export.bats +function( pkg, base ) + std.mergePatch(base, std.get({ + WireCellGen: { + MultiDuctor: { + config: { + fields: base.MultiDuctor.config.fields + [ + { // a complex sequence of objects + name: "chains", + item: { + schema: "sequence" + }, + }, + ] + } + }, + WireBoundedDepos: { + config: { + fields: base.WireBoundedDepos.config.fields + [ + { // a complex sequence of objects + name: "regions", + item: { + schema: "sequence", + }, + }, + ], + }, + }, + }, + }, pkg, {})) + diff --git a/apps/test/set-diff.jsonnet b/apps/test/set-diff.jsonnet new file mode 100644 index 000000000..871a16f4d --- /dev/null +++ b/apps/test/set-diff.jsonnet @@ -0,0 +1,6 @@ +// Return object/set difference: big - small +function(small, big) + {[k]:big[k] for k in std.setDiff(std.objectFields(big), std.objectFields(small))} + + + diff --git a/apps/test/test-config-export.bats b/apps/test/test-config-export.bats new file mode 100644 index 000000000..088ee3340 --- /dev/null +++ b/apps/test/test-config-export.bats @@ -0,0 +1,51 @@ +#!/usr/bin/env bats +bats_load_library wct-bats.sh + +@test "config export dump schema" { + cd_tmp file + + mkdir -p raw log cfg uniq schema + + local cleanup="$(relative_path set-diff.jsonnet)" + + for libpath in $(find "$(blddir)" -name 'libWireCell*') + do + local libname="$(basename $libpath .so)" + local name=${libname#"lib"} + if [ "$name" == "WireCellApps" ] ; then + extra="" + else + extra="-p $name" + fi + local out="raw/${name}.json" + echo "[{type:\"ConfigSchema\",data:{filename: \"$out\"}}]" > \ + "cfg/${name}.jsonnet" + + wire-cell -l "log/${name}.log" -L debug -a ConfigSchema \ + -p WireCellApps $extra "cfg/${name}.jsonnet" + + done + + # subtract the components provided by WireCellApps + for one in raw/*.json + do + fname="$(basename $one)" + if [ "$fname" == "WireCellApps.json" ] ; then + continue; + fi + jsonnet -o "uniq/${fname}" \ + --tla-code-file small=raw/WireCellApps.json \ + --tla-code-file big=raw/${fname} \ + "$cleanup" + done + cp raw/WireCellApps.json uniq/ + + # patch in missing schema + local patchup="$(relative_path add-missing-schema.jsonnet)" + for one in uniq/*.json + do + pkg="$(basename $one .json)" + jsonnet -o "schema/${pkg}.json" \ + --tla-code-file base="$one" -A pkg=$pkg $patchup + done +} diff --git a/util/docs/namedfactory-classes.plantuml b/util/docs/namedfactory-classes.plantuml new file mode 100644 index 000000000..0f9231170 --- /dev/null +++ b/util/docs/namedfactory-classes.plantuml @@ -0,0 +1,52 @@ +@startuml + +class Interface +class "IComponent" as IComponent_t +class "IComponent" as IComponent_factory +class "IComponent" as IComponent_A +class "IComponent" as IComponent_B +class IFactory +class INamedFactory +class Concrete + +Interface <|-- IComponent_t +IComponent_t <|-- IComponent_factory + +IComponent_factory <|-- IFactory +IFactory <|-- INamedFactory + + +class "NamedFactory" as NamedFactory_Concrete +class NamedFactory_Concrete { + Interface::pointer find(name); + Interface::pointer create(name); + + +} + +class "NamedFactoryRegistry" as NamedFactoryRegistry_t +class NamedFactoryRegistry_t { + size_t hello(wctname, cppname, intname); + const std::string& interface_name(); + known_type_set known_types(); + wct_to_cpp_map known_classes(); + bool associate(classname, factory); + factory_ptr lookup_factory(classname); + interface_ptr instance(classname, instname, create, nullok) + std::vector known_classes() +} + +IComponent_t <|-- IComponent_A +IComponent_t <|-- IComponent_B + +IComponent_A <|-- InterfaceA +IComponent_B <|-- InterfaceB + +InterfaceA <|-- Concrete +InterfaceB <|-- Concrete + +INamedFactory <|-- NamedFactory_Concrete + + + +@enduml diff --git a/util/inc/WireCellUtil/NamedFactory.h b/util/inc/WireCellUtil/NamedFactory.h index 20e113ae8..6a89cc888 100644 --- a/util/inc/WireCellUtil/NamedFactory.h +++ b/util/inc/WireCellUtil/NamedFactory.h @@ -8,6 +8,7 @@ #include "WireCellUtil/Type.h" #include "WireCellUtil/String.h" #include "WireCellUtil/Exceptions.h" +#include "WireCellUtil/TupleHelpers.h" #include "WireCellUtil/Logging.h" #include @@ -21,6 +22,7 @@ namespace WireCell { struct FactoryException : virtual public Exception { }; + /** A templated factory of objects of type Type that associates a * name to an object, returning a preexisting one if it exists. */ template @@ -68,31 +70,65 @@ namespace WireCell { std::string m_classname; }; + class NamedFactoryRegistryBase { + public: + + struct TypeInfo { + std::string cppname; + std::set intnames; + }; + using TypeInfoMap = std::map; + + static + const TypeInfoMap& known_type_info() { return m_typeinfo; } + + protected: + + inline static TypeInfoMap m_typeinfo{}; + }; + /** A registry of factories that produce instances which implement * a given interface. */ template - class NamedFactoryRegistry { + class NamedFactoryRegistry : public NamedFactoryRegistryBase { Log::logptr_t l; - public: + public: typedef IType interface_type; typedef std::shared_ptr interface_ptr; typedef WireCell::INamedFactory* factory_ptr; typedef std::unordered_map factory_lookup; + + // The known WCT types typedef std::set known_type_set; NamedFactoryRegistry() : l(Log::logger("factory")) { } - size_t hello(const std::string& classname) + + size_t hello(const std::string& wctname, const std::string& cppname, + const std::string& intname) { - m_known_types.insert(classname); + m_known_types.insert(wctname); + + m_typeinfo[wctname].cppname = cppname; + m_typeinfo[wctname].intnames.insert(intname); + + m_interface_name = intname; return m_known_types.size(); } - known_type_set known_types() const { return m_known_types; } - /// Register an existing factory by the "class" name of the instance it can create. + const std::string& interface_name() const { + return m_interface_name; + } + + known_type_set known_types() const { + return m_known_types; + } + + /// Register an existing factory by the "class" name of the instance it + /// can create. bool associate(const std::string& classname, factory_ptr factory) { m_lookup[classname] = factory; @@ -209,6 +245,7 @@ namespace WireCell { private: factory_lookup m_lookup; known_type_set m_known_types; + std::string m_interface_name; }; /// Singleton interface @@ -324,7 +361,7 @@ namespace WireCell { } // namespace WireCell template -void* make_named_factory_factory(std::string name) +void* make_named_factory_factory(const std::string& name) { static void* void_factory = nullptr; if (!void_factory) { @@ -335,15 +372,40 @@ void* make_named_factory_factory(std::string name) return void_factory; } -template -size_t namedfactory_hello(std::string name) +template +size_t namedfactory_hello(const std::string& name, + const std::string& concrete, + std::vector interfaces) +{ + return 0; +} + +template +size_t namedfactory_hello(const std::string& name, + const std::string& concrete, + std::vector interfaces) { - std::vector ret{WireCell::Singleton >::Instance().hello(name)...}; - return ret.size(); + const size_t number = WireCell::Singleton >::Instance().hello(name, concrete, interfaces.back()); + interfaces.pop_back(); + if (interfaces.empty()) { + return number; + } + return number + namedfactory_hello(name, concrete, interfaces); +} + +inline +std::vector namedfactory_parse_reverse(const std::string& lst) +{ + auto ret = WireCell::String::split(lst, ","); + for (size_t ind=0; ind(#NAME); \ + static size_t hello_##NAME##_me = namedfactory_hello(#NAME, #CONCRETE, namedfactory_parse_reverse( #__VA_ARGS__ )); \ extern "C" { \ void* make_##NAME##_factory() { return make_named_factory_factory(#NAME); } \ } diff --git a/util/inc/WireCellUtil/String.h b/util/inc/WireCellUtil/String.h index cd8b3495a..37e31154d 100644 --- a/util/inc/WireCellUtil/String.h +++ b/util/inc/WireCellUtil/String.h @@ -27,6 +27,9 @@ namespace WireCell { std::vector split(const std::string& in, const std::string& delim = ":"); + // Return s with any leading or trailing white space removed. + std::string strip(std::string s); + std::pair parse_pair(const std::string& in, const std::string& delim = ":"); // format_flatten converts from "%"-list to variadic function call. diff --git a/util/src/String.cxx b/util/src/String.cxx index 5b5bad935..8b17f12a2 100644 --- a/util/src/String.cxx +++ b/util/src/String.cxx @@ -11,6 +11,27 @@ std::vector WireCell::String::split(const std::string& in, const st return chunks; } +std::string WireCell::String::strip(std::string s) +{ + size_t beg = 0; + size_t end = s.size(); + + while (beg < end && (s[beg] == ' ' || s[beg] == '\t' || s[beg] == '\n')) { + ++beg; + } + if (beg == end) { + return ""; + } + --end; + while (end > beg && (s[end] == ' ' || s[end] == '\t' || s[end] == '\n')) { + --end; + } + if (beg == end) { + return ""; + } + return s.substr(beg, end+1-beg); +} + std::pair WireCell::String::parse_pair(const std::string& in, const std::string& delim) { std::vector chunks = split(in, delim); diff --git a/util/test/doctest_string.cxx b/util/test/doctest_string.cxx new file mode 100644 index 000000000..1aaedcea8 --- /dev/null +++ b/util/test/doctest_string.cxx @@ -0,0 +1,19 @@ +#include "WireCellUtil/Logging.h" +#include "WireCellUtil/String.h" + +#include "WireCellUtil/doctest.h" + +using namespace WireCell::String; + +TEST_CASE("string strip") { + + CHECK(strip("") == ""); + CHECK(strip(" ") == ""); + CHECK(strip("blah") == "blah"); + CHECK(strip("blah ") == "blah"); + CHECK(strip(" blah") == "blah"); + CHECK(strip(" blah ") == "blah"); + CHECK(strip(" \t\nblah\n\t \t\n") == "blah"); + CHECK(strip(" \t\nblah blah\n\t \t\n") == "blah blah"); +} +