fixes #601 -- implement native histogram support for metrics-exporter-prometheus

Author: alex

metrics-exporter-prometheus/Cargo.toml

@@ -67,6 +67,10 @@ tracing-subscriber = { workspace = true, features = ["fmt"] }
 [build-dependencies]
 prost-build = { workspace = true, optional = true }
 
+[[example]]
+name = "native_histograms"
+required-features = ["http-listener"]
+
 [[example]]
 name = "prometheus_push_gateway"
 required-features = ["push-gateway"]

metrics-exporter-prometheus/examples/native_histograms.rs

@@ -0,0 +1,53 @@
+use metrics::histogram;
+use metrics_exporter_prometheus::{NativeHistogramConfig, PrometheusBuilder};
+use std::thread;
+use std::time::Duration;
+
+fn main() {
+    // Create a Prometheus builder with native histograms enabled for specific metrics
+    let builder = PrometheusBuilder::new()
+        .with_http_listener(([127, 0, 0, 1], 9000))
+        // Enable native histograms for specific request_duration metrics
+        .set_native_histogram_for_metric(
+            metrics_exporter_prometheus::Matcher::Prefix("request_duration_api".to_string()),
+            NativeHistogramConfig::new(1.1, 160, 1e-9).unwrap(), // Finer granularity
+        )
+        .set_native_histogram_for_metric(
+            metrics_exporter_prometheus::Matcher::Prefix("response_size".to_string()),
+            NativeHistogramConfig::new(1.1, 160, 1e-9).unwrap(), // Finer granularity
+        );
+
+    // Install the recorder and get a handle
+    builder.install().expect("failed to install recorder");
+
+    // Simulate some metric recording in a loop
+    println!("Recording metrics... Check http://127.0.0.1:9000/metrics");
+    println!("Native histograms will only be visible in protobuf format.");
+    println!(
+        "Try: curl -H 'Accept: application/vnd.google.protobuf' http://127.0.0.1:9000/metrics"
+    );
+
+    for i in 0..1000 {
+        // Record to native histogram (request_duration_api)
+        let duration = (i as f64 / 10.0).sin().abs() * 5.0 + 0.1;
+        histogram!("request_duration_api").record(duration);
+
+        // Record to regular histogram (response_size)
+        let size = 1000.0 + (i as f64).cos() * 500.0;
+        histogram!("response_size").record(size);
+
+        if i % 100 == 0 {
+            println!("Recorded {} samples", i + 1);
+        }
+
+        thread::sleep(Duration::from_millis(10));
+    }
+
+    println!("Metrics server will continue running. Access http://127.0.0.1:9000/metrics");
+    println!("Press Ctrl+C to exit");
+
+    // Keep the server running
+    loop {
+        thread::sleep(Duration::from_secs(1));
+    }
+}

metrics-exporter-prometheus/src/distribution.rs

@@ -5,6 +5,7 @@ use std::{collections::HashMap, sync::Arc};
 use quanta::Instant;
 
 use crate::common::Matcher;
+use crate::native_histogram::{NativeHistogram, NativeHistogramConfig};
 
 use metrics_util::{
     storage::{Histogram, Summary},
@@ -32,6 +33,11 @@ pub enum Distribution {
     /// requests were faster than 200ms, and 99% of requests were faster than
     /// 1000ms, etc.
     Summary(RollingSummary, Arc<Vec<Quantile>>, f64),
+    /// A Prometheus native histogram.
+    ///
+    /// Uses exponential buckets to efficiently represent histogram data without
+    /// requiring predefined bucket boundaries.
+    NativeHistogram(NativeHistogram),
 }
 
 impl Distribution {
@@ -54,6 +60,12 @@ impl Distribution {
         Distribution::Summary(RollingSummary::new(bucket_count, bucket_duration), quantiles, 0.0)
     }
 
+    /// Creates a native histogram distribution.
+    pub fn new_native_histogram(config: NativeHistogramConfig) -> Distribution {
+        let hist = NativeHistogram::new(config);
+        Distribution::NativeHistogram(hist)
+    }
+
     /// Records the given `samples` in the current distribution.
     pub fn record_samples(&mut self, samples: &[(f64, Instant)]) {
         match self {
@@ -66,6 +78,11 @@ impl Distribution {
                     *sum += *sample;
                 }
             }
+            Distribution::NativeHistogram(hist) => {
+                for (sample, _ts) in samples {
+                    hist.observe(*sample);
+                }
+            }
         }
     }
 }
@@ -78,6 +95,7 @@ pub struct DistributionBuilder {
     bucket_duration: Option<Duration>,
     bucket_count: Option<NonZeroU32>,
     bucket_overrides: Option<Vec<(Matcher, Vec<f64>)>>,
+    native_histogram_overrides: Option<Vec<(Matcher, NativeHistogramConfig)>>,
 }
 
 impl DistributionBuilder {
@@ -88,6 +106,7 @@ impl DistributionBuilder {
         buckets: Option<Vec<f64>>,
         bucket_count: Option<NonZeroU32>,
         bucket_overrides: Option<HashMap<Matcher, Vec<f64>>>,
+        native_histogram_overrides: Option<HashMap<Matcher, NativeHistogramConfig>>,
     ) -> DistributionBuilder {
         DistributionBuilder {
             quantiles: Arc::new(quantiles),
@@ -99,11 +118,26 @@ impl DistributionBuilder {
                 matchers.sort_by(|a, b| a.0.cmp(&b.0));
                 matchers
             }),
+            native_histogram_overrides: native_histogram_overrides.map(|entries| {
+                let mut matchers = entries.into_iter().collect::<Vec<_>>();
+                matchers.sort_by(|a, b| a.0.cmp(&b.0));
+                matchers
+            }),
         }
     }
 
     /// Returns a distribution for the given metric key.
     pub fn get_distribution(&self, name: &str) -> Distribution {
+        // Check for native histogram overrides first (highest priority)
+        if let Some(ref overrides) = self.native_histogram_overrides {
+            for (matcher, config) in overrides {
+                if matcher.matches(name) {
+                    return Distribution::new_native_histogram(config.clone());
+                }
+            }
+        }
+
+        // Check for histogram bucket overrides
         if let Some(ref overrides) = self.bucket_overrides {
             for (matcher, buckets) in overrides {
                 if matcher.matches(name) {
@@ -112,10 +146,12 @@ impl DistributionBuilder {
             }
         }
 
+        // Check for global histogram buckets
         if let Some(ref buckets) = self.buckets {
             return Distribution::new_histogram(buckets);
         }
 
+        // Default to summary
         let b_duration = self.bucket_duration.map_or(DEFAULT_SUMMARY_BUCKET_DURATION, |d| d);
         let b_count = self.bucket_count.map_or(DEFAULT_SUMMARY_BUCKET_COUNT, |c| c);
 
@@ -124,6 +160,16 @@ impl DistributionBuilder {
 
     /// Returns the distribution type for the given metric key.
     pub fn get_distribution_type(&self, name: &str) -> &'static str {
+        // Check for native histogram overrides first (highest priority)
+        if let Some(ref overrides) = self.native_histogram_overrides {
+            for (matcher, _) in overrides {
+                if matcher.matches(name) {
+                    return "native_histogram";
+                }
+            }
+        }
+
+        // Check for regular histogram buckets
         if self.buckets.is_some() {
             return "histogram";
         }

metrics-exporter-prometheus/src/exporter/builder.rs

@@ -24,6 +24,7 @@ use metrics_util::{
 
 use crate::common::Matcher;
 use crate::distribution::DistributionBuilder;
+use crate::native_histogram::NativeHistogramConfig;
 use crate::recorder::{Inner, PrometheusRecorder};
 use crate::registry::AtomicStorage;
 use crate::{common::BuildError, PrometheusHandle};
@@ -44,6 +45,7 @@ pub struct PrometheusBuilder {
     bucket_count: Option<NonZeroU32>,
     buckets: Option<Vec<f64>>,
     bucket_overrides: Option<HashMap<Matcher, Vec<f64>>>,
+    native_histogram_overrides: Option<HashMap<Matcher, NativeHistogramConfig>>,
     idle_timeout: Option<Duration>,
     upkeep_timeout: Duration,
     recency_mask: MetricKindMask,
@@ -79,6 +81,7 @@ impl PrometheusBuilder {
             bucket_count: None,
             buckets: None,
             bucket_overrides: None,
+            native_histogram_overrides: None,
             idle_timeout: None,
             upkeep_timeout,
             recency_mask: MetricKindMask::NONE,
@@ -345,6 +348,26 @@ impl PrometheusBuilder {
         Ok(self)
     }
 
+    /// Sets native histogram configuration for a specific pattern.
+    ///
+    /// The match pattern can be a full match (equality), prefix match, or suffix match.  The matchers are applied in
+    /// that order if two or more matchers would apply to a single metric.  That is to say, if a full match and a prefix
+    /// match applied to a metric, the full match would win, and if a prefix match and a suffix match applied to a
+    /// metric, the prefix match would win.
+    ///
+    /// Native histograms use exponential buckets and take precedence over regular histograms and summaries.
+    /// They are only supported in the protobuf format.
+    #[must_use]
+    pub fn set_native_histogram_for_metric(
+        mut self,
+        matcher: Matcher,
+        config: NativeHistogramConfig,
+    ) -> Self {
+        let overrides = self.native_histogram_overrides.get_or_insert_with(HashMap::new);
+        overrides.insert(matcher.sanitized(), config);
+        self
+    }
+
     /// Sets the idle timeout for metrics.
     ///
     /// If a metric hasn't been updated within this timeout, it will be removed from the registry and in turn removed
@@ -554,6 +577,7 @@ impl PrometheusBuilder {
                 self.buckets,
                 self.bucket_count,
                 self.bucket_overrides,
+                self.native_histogram_overrides,
             ),
             descriptions: RwLock::new(HashMap::new()),
             global_labels: self.global_labels.unwrap_or_default(),

metrics-exporter-prometheus/src/lib.rs

@@ -124,6 +124,9 @@ pub use self::common::{BuildError, Matcher};
 mod distribution;
 pub use distribution::{Distribution, DistributionBuilder};
 
+mod native_histogram;
+pub use native_histogram::{NativeHistogram, NativeHistogramConfig};
+
 mod exporter;
 pub use self::exporter::builder::PrometheusBuilder;
 #[cfg(any(feature = "http-listener", feature = "push-gateway"))]