experiment: memory profiling via uprobes

app/base-env/Dockerfile

@@ -70,6 +70,7 @@ RUN apt-get update \
   software-properties-common \
   ssh-client \
   subversion \
+  systemtap-sdt-dev \
   unzip \
   wget \
   zlib1g-dev
@@ -145,14 +146,23 @@ RUN VERSION="1.17.0" \
   && rm -rf "googletest-${VERSION}" "${TAR_NAME}"
 
 # More recent Cppcheck (ubuntu defaults to a 1.8 version)
-RUN VERSION="2.18.0" \
+# cppcheck 2.18.0 requires Python 3.7+, so use older version for Ubuntu 18
+# Older cppcheck needs CMAKE_POLICY_VERSION_MINIMUM for CMake 4.x compatibility
+RUN if [ "${UBUNTU_VERSION}" -ge 20 ]; then \
+      VERSION="2.18.0"; \
+      SHA256="dc74e300ac59f2ef9f9c05c21d48ae4c8dd1ce17f08914dd30c738ff482e748f"; \
+      CMAKE_COMPAT_FLAG=""; \
+    else \
+      VERSION="2.13.0"; \
+      SHA256="8229afe1dddc3ed893248b8a723b428dc221ea014fbc76e6289840857c03d450"; \
+      CMAKE_COMPAT_FLAG="-DCMAKE_POLICY_VERSION_MINIMUM=3.5"; \
+    fi \
   && TAR_NAME="${VERSION}.tar.gz" \
   && curl -fsSLO "https://github.com/danmar/cppcheck/archive/refs/tags/${TAR_NAME}" \
-  && SHA256="dc74e300ac59f2ef9f9c05c21d48ae4c8dd1ce17f08914dd30c738ff482e748f" \
   && (printf "${SHA256} ${TAR_NAME}" | sha256sum --check --strict --status) \
   && tar xf "${TAR_NAME}" \
   && pushd "cppcheck-${VERSION}" \
-  && cmake -GNinja -Bbuild -DCMAKE_BUILD_TYPE=Release \
+  && cmake -GNinja -Bbuild -DCMAKE_BUILD_TYPE=Release ${CMAKE_COMPAT_FLAG} \
   && cmake --build build -t install \
   && popd \
   && rm -rf "cppcheck-${VERSION}" "${TAR_NAME}"

include/ddprof_cli.hpp

@@ -78,6 +78,10 @@ struct DDProfCLI {
   bool continue_exec{false};
   bool timeline{true};
 
+  // SDT probe options
+  std::string sdt_mode{"auto"}; // auto, only, off
+  std::string target_binary;    // Path for SDT probe discovery
+
   // args
   std::vector<std::string> command_line;
 

include/ddprof_context.hpp

@@ -12,7 +12,9 @@
 #include "exporter_input.hpp"
 #include "perf_clock.hpp"
 #include "perf_watcher.hpp"
+#include "sdt_allocation_correlator.hpp"
 #include "unique_fd.hpp"
+#include "uprobe_attacher.hpp"
 
 #include <sched.h>
 #include <unistd.h>
@@ -44,6 +46,10 @@ struct DDProfContext {
     std::string tags;
     std::chrono::milliseconds initial_loaded_libs_check_delay{0};
     std::chrono::milliseconds loaded_libs_check_interval{0};
+
+    // SDT probe options
+    std::string sdt_mode{"auto"};
+    std::string target_binary;
   } params;
 
   ddprof::UniqueFd socket_fd;
@@ -52,5 +58,10 @@ struct DDProfContext {
   std::vector<PerfWatcher> watchers;
   ExporterInput exp_input;
   DDProfWorkerContext worker_ctx;
+
+  // SDT probe support
+  std::vector<UprobeAttachment> sdt_attachments; // Attached uprobes for SDT
+  SDTAllocationCorrelator sdt_correlator;        // Entry/exit correlation
+  bool sdt_probes_active{false}; // True if SDT probes are being used
 };
 } // namespace ddprof

include/ddprof_context_lib.hpp

@@ -18,4 +18,9 @@ DDRes context_set(const DDProfCLI &ddprof_cli, DDProfContext &ctx);
 
 int context_allocation_profiling_watcher_idx(const DDProfContext &ctx);
 
+int context_sdt_allocation_profiling_watcher_idx(const DDProfContext &ctx);
+
+DDRes context_setup_sdt_probes(const DDProfCLI &ddprof_cli,
+                               DDProfContext &ctx);
+
 } // namespace ddprof

include/ddprof_worker.hpp

@@ -11,6 +11,7 @@
 #include "ddres.hpp"
 #include "persistent_worker_state.hpp"
 #include "pevent.hpp"
+#include "sdt_probe.hpp"
 
 namespace ddprof {
 struct DDProfContext;
@@ -24,7 +25,8 @@ DDRes ddprof_worker_cycle(DDProfContext &ctx,
                           std::chrono::steady_clock::time_point now,
                           bool synchronous_export);
 DDRes ddprof_worker_process_event(const perf_event_header *hdr, int watcher_pos,
-                                  DDProfContext &ctx);
+                                  DDProfContext &ctx,
+                                  SDTProbeType sdt_probe_type = SDTProbeType::kUnknown);
 
 // Only init unwinding elements
 DDRes worker_library_init(DDProfContext &ctx,

include/perf_watcher.hpp

@@ -98,10 +98,14 @@ enum DDPROF_SAMPLE_TYPES : uint8_t {
 #undef X_ENUM
 
 // Define our own event type on top of perf event types
-enum DDProfTypeId : uint8_t { kDDPROF_TYPE_CUSTOM = PERF_TYPE_MAX + 100 };
+enum DDProfTypeId : uint8_t {
+  kDDPROF_TYPE_CUSTOM = PERF_TYPE_MAX + 100,
+  kDDPROF_TYPE_SDT_UPROBE = PERF_TYPE_MAX + 101,
+};
 
 enum DDProfCustomCountId : uint8_t {
   kDDPROF_COUNT_ALLOCATIONS = 0,
+  kDDPROF_COUNT_ALLOCATIONS_SDT = 1,
 };
 
 // Kernel events are necessary to get a full accounting of CPU
@@ -122,6 +126,9 @@ enum DDProfCustomCountId : uint8_t {
 
 #define SKIP_FRAMES {.nb_frames_to_skip = NB_FRAMES_TO_SKIP}
 
+// SDT probes don't add frames to the stack (uprobe fires at probe location)
+#define SDT_NO_SKIP {.nb_frames_to_skip = 0}
+
 // Whereas tracepoints are dynamically configured and can be checked at runtime,
 // we lack the ability to inspect events of type other than TYPE_TRACEPOINT.
 // Accordingly, we maintain a list of events, even though the type of these
@@ -150,7 +157,8 @@ enum DDProfCustomCountId : uint8_t {
   X(sALGN,      "Align. Faults",      PERF_TYPE_SOFTWARE,   PERF_COUNT_SW_ALIGNMENT_FAULTS,        99,           DDPROF_PWT_TRACEPOINT,  IS_FREQ)                 \
   X(sEMU,       "Emu. Faults",        PERF_TYPE_SOFTWARE,   PERF_COUNT_SW_EMULATION_FAULTS,        99,           DDPROF_PWT_TRACEPOINT,  IS_FREQ)                 \
   X(sDUM,       "Dummy",              PERF_TYPE_SOFTWARE,   PERF_COUNT_SW_DUMMY,                   1,            DDPROF_PWT_NOCOUNT,     {})                      \
-  X(sALLOC,     "Allocations",        kDDPROF_TYPE_CUSTOM,  kDDPROF_COUNT_ALLOCATIONS,             524288,       DDPROF_PWT_ALLOC_SPACE, SKIP_FRAMES)
+  X(sALLOC,     "Allocations",        kDDPROF_TYPE_CUSTOM,  kDDPROF_COUNT_ALLOCATIONS,             524288,       DDPROF_PWT_ALLOC_SPACE, SKIP_FRAMES)             \
+  X(sALLOC_SDT, "SDT Allocations",   kDDPROF_TYPE_SDT_UPROBE, kDDPROF_COUNT_ALLOCATIONS_SDT,      524288,       DDPROF_PWT_ALLOC_SPACE, SDT_NO_SKIP)
 
 // clang-format on
 

include/pevent.hpp

@@ -7,6 +7,7 @@
 
 #include "ddprof_defs.hpp"
 #include "perf_ringbuffer.hpp"
+#include "sdt_probe.hpp"
 
 #include <sys/types.h>
 
@@ -29,6 +30,8 @@ struct PEvent {
   std::vector<int>
       sub_fds; // perf FDs of other events outputting to the same ring buffer
                // (eg. perf events for other process threads in PID mode)
+  SDTProbeType sdt_probe_type{
+      SDTProbeType::kUnknown}; // Type of SDT probe (for uprobe events)
 };
 
 struct PEventHdr {

include/sdt_allocation_correlator.hpp

@@ -0,0 +1,127 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include "unwind_output.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <unordered_map>
+#include <utility>
+
+namespace ddprof {
+
+/// Represents a pending malloc that hasn't been matched with its exit yet
+struct PendingAllocation {
+  pid_t pid;
+  pid_t tid;
+  uint64_t size;
+  uint64_t entry_timestamp;
+  UnwindOutput stack; // Stack trace captured at malloc entry
+};
+
+/// Result of a successful malloc correlation
+struct CorrelatedAllocation {
+  uint64_t size;       // Allocation size from entry
+  uintptr_t ptr;       // Returned pointer from exit
+  UnwindOutput stack;  // Stack trace from entry
+  uint64_t timestamp;  // Exit timestamp
+};
+
+/// Correlates malloc entry (size) with exit (pointer) events
+/// 
+/// When malloc is called:
+/// 1. Entry probe fires with size argument -> on_malloc_entry() stores it
+/// 2. Exit probe fires with returned pointer -> on_malloc_exit() correlates
+/// 
+/// This class maintains per-thread pending allocations and matches them.
+class SDTAllocationCorrelator {
+public:
+  SDTAllocationCorrelator();
+  ~SDTAllocationCorrelator();
+
+  // Non-copyable, movable
+  SDTAllocationCorrelator(const SDTAllocationCorrelator &) = delete;
+  SDTAllocationCorrelator &operator=(const SDTAllocationCorrelator &) = delete;
+  SDTAllocationCorrelator(SDTAllocationCorrelator &&) = default;
+  SDTAllocationCorrelator &operator=(SDTAllocationCorrelator &&) = default;
+
+  /// Record a malloc entry event
+  /// @param pid Process ID
+  /// @param tid Thread ID
+  /// @param size Allocation size
+  /// @param timestamp Event timestamp
+  /// @param stack Stack trace at entry
+  void on_malloc_entry(pid_t pid, pid_t tid, uint64_t size, uint64_t timestamp,
+                       UnwindOutput stack);
+
+  /// Process a malloc exit event and try to correlate with entry
+  /// @param pid Process ID
+  /// @param tid Thread ID
+  /// @param ptr Returned pointer
+  /// @param timestamp Event timestamp
+  /// @return Correlated allocation if successful, nullopt if no matching entry
+  std::optional<CorrelatedAllocation> on_malloc_exit(pid_t pid, pid_t tid,
+                                                     uintptr_t ptr,
+                                                     uint64_t timestamp);
+
+  /// Record a free entry event (for deallocation tracking)
+  /// This is a pass-through - just returns the info needed to track deallocation
+  /// @param pid Process ID
+  /// @param tid Thread ID
+  /// @param ptr Pointer being freed
+  /// @param timestamp Event timestamp
+  void on_free_entry(pid_t pid, pid_t tid, uintptr_t ptr, uint64_t timestamp);
+
+  /// Clean up stale pending entries that are older than max_age_ns
+  /// Call this periodically to prevent memory leaks from lost events
+  /// @param current_time Current timestamp
+  /// @param max_age_ns Maximum age in nanoseconds
+  /// @return Number of stale entries cleaned up
+  size_t cleanup_stale(uint64_t current_time, uint64_t max_age_ns);
+
+  /// Get the number of pending (unmatched) allocations
+  size_t pending_count() const { return _pending.size(); }
+
+  /// Get statistics
+  uint64_t total_entries() const { return _total_entries; }
+  uint64_t total_exits() const { return _total_exits; }
+  uint64_t successful_correlations() const { return _successful_correlations; }
+  uint64_t missed_entries() const { return _missed_entries; }
+  uint64_t missed_exits() const { return _missed_exits; }
+  uint64_t stale_cleanups() const { return _stale_cleanups; }
+
+  /// Reset statistics
+  void reset_stats();
+
+  /// Default maximum correlation age (1 second)
+  static constexpr uint64_t kDefaultMaxCorrelationAge = 1'000'000'000ULL;
+
+private:
+  /// Hash function for (pid, tid) pair
+  struct TidHash {
+    size_t operator()(const std::pair<pid_t, pid_t> &p) const {
+      // Combine pid and tid into a single hash
+      return std::hash<uint64_t>{}((static_cast<uint64_t>(p.first) << 32) |
+                                   static_cast<uint32_t>(p.second));
+    }
+  };
+
+  // Key: (pid, tid), Value: pending allocation
+  // Each thread can have at most one pending malloc at a time
+  std::unordered_map<std::pair<pid_t, pid_t>, PendingAllocation, TidHash>
+      _pending;
+
+  // Statistics
+  uint64_t _total_entries{0};
+  uint64_t _total_exits{0};
+  uint64_t _successful_correlations{0};
+  uint64_t _missed_entries{0}; // Exit without matching entry
+  uint64_t _missed_exits{0};   // Entry overwritten before exit
+  uint64_t _stale_cleanups{0}; // Entries cleaned up due to age
+};
+
+} // namespace ddprof

include/sdt_probe.hpp

@@ -0,0 +1,102 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace ddprof {
+
+/// Location type for SDT probe arguments
+enum class SDTArgLocation : uint8_t {
+  kRegister, // Value in a CPU register
+  kMemory,   // Value at memory location (base register + offset)
+  kConstant, // Constant/immediate value
+};
+
+/// Represents a single argument to an SDT probe
+/// Arguments are encoded as assembly expressions like "8@%rdi" or "-4@8(%rbp)"
+struct SDTArgument {
+  int8_t size;          // Size in bytes (negative = signed)
+  SDTArgLocation location;
+  uint8_t base_reg;     // Base register number (for register or memory)
+  uint8_t index_reg;    // Index register (for scaled memory addressing)
+  uint8_t scale;        // Scale factor for index register
+  int64_t offset;       // Memory offset or constant value
+  std::string raw_spec; // Original argument specification
+};
+
+/// Represents a discovered SDT probe from .note.stapsdt section
+struct SDTProbe {
+  std::string provider; // e.g., "ddprof_malloc"
+  std::string name;     // e.g., "entry"
+  uint64_t address;     // Probe location (virtual address in ELF)
+  uint64_t base;        // Base address for prelink adjustment
+  uint64_t semaphore;   // Semaphore address (0 if not used)
+  std::vector<SDTArgument> arguments;
+
+  /// Get full probe name as "provider:name"
+  std::string full_name() const { return provider + ":" + name; }
+};
+
+/// Probe types for memory allocation tracking
+enum class SDTProbeType : uint8_t {
+  kUnknown,
+  kMallocEntry,
+  kMallocExit,
+  kFreeEntry,
+  kFreeExit,
+};
+
+/// Collection of SDT probes discovered from a binary
+struct SDTProbeSet {
+  std::string binary_path;
+  std::vector<SDTProbe> probes;
+
+  /// Find probes matching a provider and name
+  std::vector<const SDTProbe *> find_probes(std::string_view provider,
+                                            std::string_view name) const;
+
+  /// Find a single probe by provider and name (returns first match)
+  const SDTProbe *find_probe(std::string_view provider,
+                             std::string_view name) const;
+
+  /// Check if the probe set contains all required allocation probes
+  bool has_allocation_probes() const;
+
+  /// Get the probe type for a probe
+  static SDTProbeType get_probe_type(const SDTProbe &probe);
+};
+
+/// Parse SDT probes from an ELF binary file
+/// Returns nullopt if no probes found or file cannot be read
+std::optional<SDTProbeSet> parse_sdt_probes(const char *filepath);
+
+/// Parse a single SDT argument specification
+/// Format: [+-]?size@location where location can be:
+///   - %reg (register)
+///   - constant (immediate value)
+///   - offset(%reg) (memory reference)
+///   - offset(%base,%index,scale) (scaled indexed addressing)
+std::optional<SDTArgument> parse_sdt_argument(std::string_view arg_spec);
+
+/// Convert x86-64 register name to perf register number
+/// Returns -1 if register name is not recognized
+int x86_reg_name_to_perf_reg(std::string_view reg_name);
+
+/// Provider name for malloc probes
+inline constexpr std::string_view kMallocProvider = "ddprof_malloc";
+/// Provider name for free probes
+inline constexpr std::string_view kFreeProvider = "ddprof_free";
+/// Entry probe name
+inline constexpr std::string_view kEntryProbe = "entry";
+/// Exit probe name
+inline constexpr std::string_view kExitProbe = "exit";
+
+} // namespace ddprof