Storage of job execution environment in output files [1/4]

CMakeLists.txt

@@ -25,6 +25,9 @@ message(STATUS
 include(CetCMakeEnv)
 cet_cmake_env()
 
+# following art's approach, our CMake stuff is in `Modules` directory:
+cet_cmake_module_directories(Modules BINARY)
+
 cet_set_compiler_flags(DIAGS CAUTIOUS
   WERROR
   NO_UNDEFINED
@@ -50,11 +53,13 @@ find_package( dk2nudata REQUIRED )
 include(ArtDictionary)
 include(CetMake)
 include(BasicPlugin)
+include(SBNutils)
 
 # add cet_find_library commands here when needed
 
 # ADD SOURCE CODE SUBDIRECTORIES HERE
 add_subdirectory(sbnobj)
+add_subdirectory(Modules)
 
 # tests
 add_subdirectory(test)

Modules/CMakeLists.txt

@@ -0,0 +1,2 @@
+install(DIRECTORY ./ DESTINATION Modules
+  FILES_MATCHING PATTERN "*.cmake" PATTERN "[.#]*.cmake" EXCLUDE)

Modules/SBNutils.cmake

@@ -0,0 +1,98 @@
+#
+# File:    SBNmacros.cmake
+# Purpose: Helpers for CMake actions in SBN.
+# Author:  Gianluca Petrillo (petrillo@slac.stanford.edu)
+# Date:    January 17, 2025
+#
+
+include_guard()
+
+cmake_minimum_required(VERSION 3.27 FATAL_ERROR)
+
+function(GetGITrepoName OutputVariableName)
+  #
+  # Gets the name of the current repository from the remote source `origin `URL.
+  # 
+  # In case of failure from GIT call, the project name will be returned.
+  #
+  set(RepoDir ${CMAKE_CURRENT_SOURCE_DIR})
+
+  execute_process(COMMAND git -C "${RepoDir}" remote get-url origin
+    OUTPUT_VARIABLE GITrepoURL OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET RESULT_VARIABLE ErrorCode
+    )
+  if(ErrorCode)
+    message(WARNING "Failed to retrieve GIT repository name for ${CMAKE_PROJECT_NAME} (error code ${ErrorCode})")
+    set(${OutputVariableName} "${CMAKE_PROJECT_NAME}")
+  else()
+    cmake_path(GET GITrepoURL STEM ${OutputVariableName})
+  endif()
+
+  return(PROPAGATE ${OutputVariableName})
+
+endfunction(GetGITrepoName)
+
+
+function(GetGITrepoVersion OutputVariableName)
+  #
+  # Gets the version/tag of the current branch in the current repository
+  # using `git describe`.
+  #
+  # In case of failure from GIT call, a error code will be returned in the
+  # variable.
+  #
+  set(RepoDir ${CMAKE_CURRENT_SOURCE_DIR})
+
+  execute_process(COMMAND git -C "${RepoDir}" describe --tags --dirty
+    OUTPUT_VARIABLE ${OutputVariableName} OUTPUT_STRIP_TRAILING_WHITESPACE
+    ERROR_QUIET RESULT_VARIABLE ErrorCode
+    )
+
+  if(ErrorCode)
+    set(${OutputVariableName} "n/a (GIT error code: ${ErrorCode})")
+  endif()
+
+  return(PROPAGATE ${OutputVariableName})
+
+endfunction(GetGITrepoVersion)
+
+
+function(GenerateRepoVersionSource OutputPrefix)
+  #
+  # GenerateRepoVersionSource [OUTPUT_PREFIX prefix]
+  #   * OUTPUT_PREFIX: prefix to the output variable names; default: ${CMAKE_PROJECT_NAME}
+  #
+  # Will generate a C++ header and source with the GIT repository describe
+  # string stored in a constant, using the `GITrepoVersion.{h,cxx}.in` template.
+  # The resulting code is written into ${CMAKE_CURRENT_BINARY_DIR} directory.
+  # In addition, it will set three variables:
+  #   <prefix>_GIT_REPO_NAME (typically matches the project name)
+  #   <prefix>_GIT_REPO_VERSION (GIT describe output)
+  #   <prefix>_GIT_REPO_VERSION_SOURCE (stem of the generated source code name);
+  #     the actual files have the suffixes `.h` and `.cxx`
+  #
+
+  if(NOT OutputPrefix)
+    set(OutputPrefix ${CMAKE_PROJECT_NAME})
+  endif()
+
+  if(OutputPrefix)
+    set(OutputPrefix "${OutputPrefix}_")
+  endif()
+
+  # we could use `${CMAKE_PROJECT_NAME}`, provided that is up to date
+  GetGITrepoName(gitRepoName)
+
+  GetGITrepoVersion(gitRepoVersion)
+
+  set(gitRepoVersionSourceStem "RepositoryVersion_${gitRepoName}")
+  message(STATUS "GIT reported repository: ${gitRepoName} ${gitRepoVersion} (=> '${gitRepoVersionSourceStem}{.h,.cxx}')")
+  configure_file("GITrepoVersion.h.in" "${gitRepoVersionSourceStem}.h")
+  configure_file("GITrepoVersion.cxx.in" "${gitRepoVersionSourceStem}.cxx")
+
+  # these are the output variables
+  set("${OutputPrefix}GIT_REPO_NAME"           "${gitRepoName}"              PARENT_SCOPE)
+  set("${OutputPrefix}GIT_REPO_VERSION"        "${gitRepoVersion}"           PARENT_SCOPE)
+  set("${OutputPrefix}GIT_REPO_VERSION_SOURCE" "${gitRepoVersionSourceStem}" PARENT_SCOPE)
+
+endfunction(GenerateRepoVersionSource)

sbnobj/CMakeLists.txt

@@ -1,5 +1,6 @@
 # basic source code CMakeLists.txt
 
+add_subdirectory(Metadata)
 add_subdirectory(Common)
 add_subdirectory(ICARUS)
 add_subdirectory(SBND)

sbnobj/Common/CMakeLists.txt

@@ -7,3 +7,4 @@ add_subdirectory(SBNEventWeight)
 add_subdirectory(POTAccounting)
 add_subdirectory(EventGen)
 add_subdirectory(Trigger)
+add_subdirectory(Metadata)

sbnobj/Common/Metadata/CMakeLists.txt

@@ -0,0 +1,13 @@
+file(GLOB lib_srcs *.cxx details/*)
+cet_make_library(
+  SOURCE
+    ${lib_srcs}
+  )
+
+art_dictionary(
+  DICTIONARY_LIBRARIES
+    sbnobj::Common_Metadata
+  )
+
+install_headers(SUBDIRS details)
+install_source()

sbnobj/Common/Metadata/JobEnvironmentInfo.cxx

@@ -0,0 +1,55 @@
+/**
+ * @file   sbnobj/Common/Metadata/JobEnvironmentInfo.cxx
+ * @brief  Information from the execution environment of the job.
+ * @author Gianluca Petrillo (petrillo@slac.stanford.edu)
+ * @date   January 16, 2025
+ * @see    sbnobj/Common/Metadata/JobEnvironmentInfo.h
+ */
+
+// library header
+#include "sbnobj/Common/Metadata/JobEnvironmentInfo.h"
+
+// C/C++ standard libraries
+#include <ostream>
+#include <sstream>
+
+
+// -----------------------------------------------------------------------------
+// ---  sbn::JobEnvironmentInfo  -----------------------------------------------
+// -----------------------------------------------------------------------------
+void sbn::JobEnvironmentInfo::dump(std::ostream& out) const {
+
+  out << "Process name: '" << processName << "'";
+
+  out << "\n\nRelease of art framework: " << artVersion;
+
+  out << "\n\nSource repository versions (" << sources.size() << ")\n"
+    << std::string(80, '-') << "\n\n";
+  sources.dump(out);
+
+  out << "\n\nEnvironment variables (" << variables.size() << ")\n"
+    << std::string(80, '-') << "\n\n";
+  variables.dump(out);
+
+} // sbn::JobEnvironmentInfo::dump()
+
+
+// -----------------------------------------------------------------------------
+std::string sbn::JobEnvironmentInfo::to_string() const {
+
+  std::ostringstream sstr;
+  dump(sstr);
+  return sstr.str();
+
+}
+
+
+// -----------------------------------------------------------------------------
+// ---  free functions  --------------------------------------------------------
+// -----------------------------------------------------------------------------
+std::ostream& sbn::operator<<
+  (std::ostream& out, JobEnvironmentInfo const& info)
+  { info.dump(out); return out; }
+
+
+// -----------------------------------------------------------------------------

sbnobj/Common/Metadata/JobEnvironmentInfo.h

@@ -0,0 +1,70 @@
+/**
+ * @file   sbnobj/Common/Metadata/JobEnvironmentInfo.h
+ * @brief  Information from the execution environment of the job.
+ * @author Gianluca Petrillo (petrillo@slac.stanford.edu)
+ * @date   January 16, 2025
+ * @see    sbnobj/Common/Metadata/JobEnvironmentInfo.cxx
+ */
+
+
+#ifndef SBNOBJ_COMMON_METADATA_JOBENVIRONMENTINFO_H
+#define SBNOBJ_COMMON_METADATA_JOBENVIRONMENTINFO_H
+
+// SBN libraries
+#include "sbnobj/Common/Metadata/OrderedPairList.h"
+
+// C/C++ standard libraries
+#include <optional>
+#include <string>
+#include <tuple> // std::pair
+#include <vector>
+#include <iosfwd> // std::ostream
+
+
+// -----------------------------------------------------------------------------
+namespace sbn {
+  struct JobEnvironmentInfo;
+
+  /// Dump of a `JobEnvironmentInfo` object into a C++ output stream.
+  std::ostream& operator<< (std::ostream& out, JobEnvironmentInfo const& info);
+
+}
+
+// -----------------------------------------------------------------------------
+/**
+ * @brief Information from the execution environment of the job.
+ * 
+ * Current information includes:
+ *  * a complete snapshot of the environment variables (`getenv()`);
+ *  * _art_ version;
+ *  * a list of GIT tag for the package sources.
+ * 
+ * Future extensions should include:
+ *  * A list of packages recognised as SciSoft stack packages, their versions,
+ *    qualifiers and paths. Probably not worth until build system is revamped.
+ * 
+ */
+struct sbn::JobEnvironmentInfo {
+
+  std::string processName; ///< Name of the _art_ process for the job.
+
+  std::string artVersion; ///< Version of _art_.
+
+  sbn::OrderedPairList sources; ///< Version of the source repositories.
+
+  sbn::OrderedPairList variables; ///< Environment variables.
+
+
+  /// Dumps the entire content of the environment in human-readable form.
+  void dump(std::ostream& out) const;
+
+  /// Returns the entire content of the environment in human-readable form.
+  std::string to_string() const;
+
+}; // sbn::JobEnvironmentInfo
+
+
+
+// -----------------------------------------------------------------------------
+
+#endif // SBNOBJ_COMMON_METADATA_JOBENVIRONMENTINFO_H

sbnobj/Common/Metadata/OrderedPairList.cxx

@@ -0,0 +1,106 @@
+/**
+ * @file   sbnobj/Common/Metadata/OrderedPairList.cxx
+ * @brief  Object holding a list of pairs.
+ * @author Gianluca Petrillo (petrillo@slac.stanford.edu)
+ * @date   January 16, 2025
+ * @see    sbnobj/Common/Metadata/OrderedPairList.h
+ */
+
+// library header
+#include "sbnobj/Common/Metadata/OrderedPairList.h"
+
+// C/C++ standard libraries
+#include <algorithm> // std::binary_search(), std::lower_bound()
+#include <ostream>
+#include <sstream>
+#include <utility> // std::less
+#include <type_traits> // std::true_type
+
+
+// -----------------------------------------------------------------------------
+// ---  sbn::OrderedPairList  ------------------------------------------
+// -----------------------------------------------------------------------------
+bool sbn::OrderedPairList::contains(Key_t const& key) const noexcept {
+
+  return std::binary_search(items.begin(), items.end(), key, ItemSorter{});
+
+}
+
+
+// -----------------------------------------------------------------------------
+auto sbn::OrderedPairList::getPtr(Key_t const& key) const -> Value_t const* {
+
+  auto const it = findInsertionPoint(key);
+  return ((it == items.end()) || (std::get<0>(*it) != key))
+    ? nullptr: &(std::get<1>(*it));
+
+}
+
+
+// -----------------------------------------------------------------------------
+auto sbn::OrderedPairList::get(Key_t const& key) const -> std::optional<Value_t>
+{
+
+  Value_t const* ptr = getPtr(key);
+  return ptr? std::make_optional(*ptr): std::nullopt;
+
+}
+
+
+// -----------------------------------------------------------------------------
+sbn::OrderedPairList& sbn::OrderedPairList::finish() {
+
+  std::sort(items.begin(), items.end());
+
+  // remove duplicates
+  auto const itFirstDuplicate = std::unique(items.begin(), items.end());
+  items.erase(itFirstDuplicate, items.end());
+
+  return *this;
+
+} // sbn::OrderedPairList::finish()
+
+
+// -----------------------------------------------------------------------------
+bool sbn::OrderedPairList::insert(Key_t key, Value_t value) {
+
+  auto const it = findInsertionPoint(key);
+  if ((it == items.end()) || (std::get<0>(*it) != key)) { // new key
+    items.emplace(it, std::move(key), std::move(value));
+    return true;
+  }
+  else {
+    // using an index since the iterator is constant
+    items[std::distance(items.cbegin(), it)].second = std::move(value);
+    return false;
+  }
+
+} // sbn::OrderedPairList::insert()
+
+
+// -----------------------------------------------------------------------------
+void sbn::OrderedPairList::dump(std::ostream& out) const {
+
+  for (auto const& [ key, value ]: items)
+    out << key << " = " << value << "\n";
+
+} // sbn::OrderedPairList::dump()
+
+
+// -----------------------------------------------------------------------------
+auto sbn::OrderedPairList::findInsertionPoint(Key_t const& key) const
+  -> const_iterator
+{
+  return std::lower_bound(items.begin(), items.end(), key, ItemSorter{});
+}
+
+
+// -----------------------------------------------------------------------------
+// ---  free functions  --------------------------------------------------------
+// -----------------------------------------------------------------------------
+std::ostream& sbn::operator<<
+  (std::ostream& out, OrderedPairList const& list)
+  { list.dump(out); return out; }
+
+
+// -----------------------------------------------------------------------------