From 31d3db01cafef450f63b3305b32d7382a6a3923f Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Mon, 15 Dec 2025 11:38:25 -0800 Subject: [PATCH 01/14] update some ddsketch fields from histogram --- lib/vector-core/src/metrics/ddsketch.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index b66f25ceb150b..5fb910b39f369 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -779,6 +779,8 @@ impl AgentDDSketch { /// ## Errors /// /// Returns an error if a bucket size is greater that `u32::MAX`. + #[allow(clippy::cast_possible_truncation)] + #[allow(clippy::cast_precision_loss)] pub fn transform_to_sketch(mut metric: Metric) -> Result { let sketch = match metric.data_mut().value_mut() { MetricValue::Distribution { samples, .. } => { @@ -788,10 +790,24 @@ impl AgentDDSketch { } Some(sketch) } - MetricValue::AggregatedHistogram { buckets, .. } => { + MetricValue::AggregatedHistogram { + buckets, + sum, + count, + .. + } => { let delta_buckets = mem::take(buckets); let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; + + let orig_sum = *sum; + let orig_count = *count; + if orig_count > 0 { + sketch.sum = orig_sum; + sketch.count = orig_count as u32; + sketch.avg = orig_sum / orig_count as f64; + } + Some(sketch) } // We can't convert from any other metric value. From be24867b958b243c41739804489e0bde0f86995c Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Mon, 15 Dec 2025 11:40:04 -0800 Subject: [PATCH 02/14] add changelog fragment --- changelog.d/dd_sketch_averages_from_histograms.fix.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/dd_sketch_averages_from_histograms.fix.md diff --git a/changelog.d/dd_sketch_averages_from_histograms.fix.md b/changelog.d/dd_sketch_averages_from_histograms.fix.md new file mode 100644 index 0000000000000..c02dd5d2d0cb8 --- /dev/null +++ b/changelog.d/dd_sketch_averages_from_histograms.fix.md @@ -0,0 +1,3 @@ +Improved accuracy of ddsketch averages by using histogram's sum / count. + +authors: tony-resendiz \ No newline at end of file From 3911f2ee14a3808306bf44a9bd4266b9304670d6 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Tue, 6 Jan 2026 08:59:49 -0800 Subject: [PATCH 03/14] formatting --- changelog.d/dd_sketch_averages_from_histograms.fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/dd_sketch_averages_from_histograms.fix.md b/changelog.d/dd_sketch_averages_from_histograms.fix.md index c02dd5d2d0cb8..e409858602600 100644 --- a/changelog.d/dd_sketch_averages_from_histograms.fix.md +++ b/changelog.d/dd_sketch_averages_from_histograms.fix.md @@ -1,3 +1,3 @@ Improved accuracy of ddsketch averages by using histogram's sum / count. -authors: tony-resendiz \ No newline at end of file +authors: tony-resendiz From 479e10ce9b300446f921f3f688c73199e33453c4 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Thu, 5 Feb 2026 10:37:45 -0800 Subject: [PATCH 04/14] don't modify the sketch if count is bigger than u32 --- lib/vector-core/src/metrics/ddsketch.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index 5fb910b39f369..f7bff040703ad 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -779,8 +779,6 @@ impl AgentDDSketch { /// ## Errors /// /// Returns an error if a bucket size is greater that `u32::MAX`. - #[allow(clippy::cast_possible_truncation)] - #[allow(clippy::cast_precision_loss)] pub fn transform_to_sketch(mut metric: Metric) -> Result { let sketch = match metric.data_mut().value_mut() { MetricValue::Distribution { samples, .. } => { @@ -794,18 +792,18 @@ impl AgentDDSketch { buckets, sum, count, - .. } => { let delta_buckets = mem::take(buckets); let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; - let orig_sum = *sum; - let orig_count = *count; - if orig_count > 0 { + if let Ok(count) = u32::try_from(*count) + && count > 0 + { + let orig_sum = *sum; + sketch.count = count; sketch.sum = orig_sum; - sketch.count = orig_count as u32; - sketch.avg = orig_sum / orig_count as f64; + sketch.avg = orig_sum / f64::from(count); } Some(sketch) From 38c8605ab4d46b6243b640f867fabb8ca9a00ada Mon Sep 17 00:00:00 2001 From: tony-resendiz <49211735+tony-resendiz@users.noreply.github.com> Date: Fri, 6 Feb 2026 09:56:36 -0800 Subject: [PATCH 05/14] Update lib/vector-core/src/metrics/ddsketch.rs Co-authored-by: Bruce Guenter --- lib/vector-core/src/metrics/ddsketch.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index f7bff040703ad..b3d2aafbe069a 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -800,10 +800,9 @@ impl AgentDDSketch { if let Ok(count) = u32::try_from(*count) && count > 0 { - let orig_sum = *sum; sketch.count = count; - sketch.sum = orig_sum; - sketch.avg = orig_sum / f64::from(count); + sketch.sum = *sum; + sketch.avg = *sum / f64::from(count); // or sketch.sum } Some(sketch) From cffc82cbca36a209747f3565f000014483d03272 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Mon, 23 Feb 2026 14:00:39 -0800 Subject: [PATCH 06/14] pr feedback --- .../dd_sketch_averages_from_histograms.fix.md | 4 +- lib/vector-core/src/metrics/ddsketch.rs | 58 ++++++++++++++++--- 2 files changed, 54 insertions(+), 8 deletions(-) diff --git a/changelog.d/dd_sketch_averages_from_histograms.fix.md b/changelog.d/dd_sketch_averages_from_histograms.fix.md index e409858602600..e02ed8ec0363c 100644 --- a/changelog.d/dd_sketch_averages_from_histograms.fix.md +++ b/changelog.d/dd_sketch_averages_from_histograms.fix.md @@ -1,3 +1,5 @@ -Improved accuracy of ddsketch averages by using histogram's sum / count. +Modified behavior of histograms (ddsketch) submitted by the Datadog metrics sink. Previously the ddsketch field average was interpolated from bucket boundaries, which could cause slight inaccuracies. Now the average will be calculated directly from the incoming histogram fields sum / count. + +Users with alerts, dashboards, or SLOs based on histogram averages should review and adjust thresholds after upgrading, as average values will settle to their correct levels. authors: tony-resendiz diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index b3d2aafbe069a..523618eb873c2 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -7,6 +7,7 @@ use ordered_float::OrderedFloat; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error}; use serde_with::{DeserializeAs, SerializeAs, serde_as}; use snafu::Snafu; +use tracing::warn; use vector_common::byte_size_of::ByteSizeOf; use vector_config::configurable_component; @@ -797,12 +798,20 @@ impl AgentDDSketch { let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; - if let Ok(count) = u32::try_from(*count) - && count > 0 - { - sketch.count = count; - sketch.sum = *sum; - sketch.avg = *sum / f64::from(count); // or sketch.sum + match u32::try_from(*count) { + Ok(c) => { + if c > 0 { + sketch.count = c; + sketch.sum = *sum; + sketch.avg = *sum / f64::from(c); + } + } + Err(_) => { + warn!( + message = "Aggregated histogram count too large to convert to u32; \ + falling back to interpolated average" + ); + } } Some(sketch) @@ -1110,7 +1119,8 @@ fn round_to_even(v: f64) -> f64 { #[cfg(test)] mod tests { use super::{AGENT_DEFAULT_EPS, AgentDDSketch, Config, MAX_KEY, round_to_even}; - use crate::event::metric::Bucket; + use crate::event::metric::{Bucket, MetricSketch}; + use crate::event::{Metric, MetricKind, MetricValue}; const FLOATING_POINT_ACCEPTABLE_ERROR: f64 = 1.0e-10; @@ -1219,6 +1229,40 @@ mod tests { assert!((end - max).abs() < FLOATING_POINT_ACCEPTABLE_ERROR); } + #[test] + fn test_transform_to_sketch_uses_histogram_sum_count() { + let buckets = vec![ + Bucket { + upper_limit: 1.0, + count: 1, + }, + Bucket { + upper_limit: 2.0, + count: 1, + }, + ]; + let histogram = Metric::new( + "histogram", + MetricKind::Absolute, + MetricValue::AggregatedHistogram { + count: 2, + sum: 3.0, + buckets, + }, + ); + let metric = AgentDDSketch::transform_to_sketch(histogram).unwrap(); + match metric.data().value() { + MetricValue::Sketch { + sketch: MetricSketch::AgentDDSketch(dd), + } => { + assert_eq!(dd.count, 2); + assert_eq!(dd.sum, 3.0); + assert_eq!(dd.avg, 1.5); + } + other => panic!("expected AgentDDSketch, got: {other:?}"), + } + } + #[test] fn test_out_of_range_buckets_error() { let mut sketch = AgentDDSketch::with_agent_defaults(); From 2068dc07c95e644e3c0cac880093d8adbc32d97c Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Mon, 23 Feb 2026 14:49:29 -0800 Subject: [PATCH 07/14] pr checks --- .github/actions/spelling/expect.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index 1d68a6978570b..824bab95f4ea3 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -517,6 +517,7 @@ SIGHUPs sinkbuilder sinknetworkbytessent slideover +SLOs smartphone Smol smtracking From 98a1bda6deccd72cdedc9256005af6979136b749 Mon Sep 17 00:00:00 2001 From: tony-resendiz <49211735+tony-resendiz@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:15:12 -0700 Subject: [PATCH 08/14] update changelog.d --- changelog.d/dd_sketch_averages_from_histograms.fix.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/dd_sketch_averages_from_histograms.fix.md b/changelog.d/dd_sketch_averages_from_histograms.fix.md index e02ed8ec0363c..da75e311346ff 100644 --- a/changelog.d/dd_sketch_averages_from_histograms.fix.md +++ b/changelog.d/dd_sketch_averages_from_histograms.fix.md @@ -1,5 +1,5 @@ -Modified behavior of histograms (ddsketch) submitted by the Datadog metrics sink. Previously the ddsketch field average was interpolated from bucket boundaries, which could cause slight inaccuracies. Now the average will be calculated directly from the incoming histogram fields sum / count. +Modified behavior of histograms (ddsketch) submitted by the Datadog metrics sink. Previously the ddsketch sum, count, and average fields were interpolated from bucket boundaries, which could cause slight inaccuracies. Now those fields will be calculated directly from the corresponding fields on the incoming histogram. Min and max fields remain interpolated. -Users with alerts, dashboards, or SLOs based on histogram averages should review and adjust thresholds after upgrading, as average values will settle to their correct levels. +Users with alerts, dashboards, or SLOs based on histogram averages should review and adjust thresholds after upgrading, as values will settle to their correct levels. authors: tony-resendiz From 371280abaf446b6dfbfe1b18b01ebf49dc3cebd9 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 09:56:26 -0700 Subject: [PATCH 09/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index 523618eb873c2..e157c820f95f1 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -798,9 +798,22 @@ impl AgentDDSketch { let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; + // Prefer exact avg from sum/count when available. + // + // NOTE: + // AggregatedHistogram -> DDSketch conversion historically relies entirely + // on bucket interpolation, which is approximate. + // + // Some upstream sources (e.g. OTLP) encode missing sums as 0.0, so we + // cannot distinguish between "unknown sum" and "actual zero sum" here. + // As a result, we only override when sum != 0.0 and fall back to + // interpolation otherwise. + // + // This preserves existing behavior for ambiguous cases while improving + // accuracy when a non-zero sum is available. match u32::try_from(*count) { Ok(c) => { - if c > 0 { + if c > 0 && *sum != 0.0 { sketch.count = c; sketch.sum = *sum; sketch.avg = *sum / f64::from(c); From 90450b23ff3618b767b493097c50a14ea0203c21 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 10:11:28 -0700 Subject: [PATCH 10/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index e157c820f95f1..d24cfbbdbfa66 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -795,6 +795,7 @@ impl AgentDDSketch { count, } => { let delta_buckets = mem::take(buckets); + let has_buckets = !delta_buckets.is_empty(); let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; @@ -813,7 +814,7 @@ impl AgentDDSketch { // accuracy when a non-zero sum is available. match u32::try_from(*count) { Ok(c) => { - if c > 0 && *sum != 0.0 { + if c > 0 && *sum != 0.0 && has_buckets { sketch.count = c; sketch.sum = *sum; sketch.avg = *sum / f64::from(c); From 647916b8169d802a5ef7e333236c52c6a2dfbd19 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 10:19:40 -0700 Subject: [PATCH 11/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index d24cfbbdbfa66..c2526798734af 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -795,7 +795,6 @@ impl AgentDDSketch { count, } => { let delta_buckets = mem::take(buckets); - let has_buckets = !delta_buckets.is_empty(); let mut sketch = AgentDDSketch::with_agent_defaults(); sketch.insert_interpolate_buckets(delta_buckets)?; @@ -814,7 +813,7 @@ impl AgentDDSketch { // accuracy when a non-zero sum is available. match u32::try_from(*count) { Ok(c) => { - if c > 0 && *sum != 0.0 && has_buckets { + if c > 0 && *sum != 0.0 && sketch.count() > 0{ sketch.count = c; sketch.sum = *sum; sketch.avg = *sum / f64::from(c); From 52d0ee4e1543b6a0afdb03dd36f5f5af01c04188 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 10:34:08 -0700 Subject: [PATCH 12/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index c2526798734af..df9f74e4ddf0c 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -814,7 +814,6 @@ impl AgentDDSketch { match u32::try_from(*count) { Ok(c) => { if c > 0 && *sum != 0.0 && sketch.count() > 0{ - sketch.count = c; sketch.sum = *sum; sketch.avg = *sum / f64::from(c); } From 5cfc57f701276e3e069544b107bd118da4a20268 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 10:52:55 -0700 Subject: [PATCH 13/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index df9f74e4ddf0c..69b32cf97e6f7 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -811,6 +811,16 @@ impl AgentDDSketch { // // This preserves existing behavior for ambiguous cases while improving // accuracy when a non-zero sum is available. + // + // Do not override sketch.count here. + // The interpolated bins define the sketch's sample mass, and count must + // remain consistent with those bins to preserve DDSketch invariants. + // + // We do override sum/avg to improve accuracy when upstream provides them. + // Note that this can result in avg != sum / sketch.count if upstream + // count differs from bucket totals (e.g. Prometheus +Inf bucket drop). + // This tradeoff favors more accurate summary stats while preserving + // correct quantile behavior. match u32::try_from(*count) { Ok(c) => { if c > 0 && *sum != 0.0 && sketch.count() > 0{ From 7a840552fa9219cd8540460ea0956281ce2bccf2 Mon Sep 17 00:00:00 2001 From: tony-resendiz Date: Sat, 2 May 2026 11:06:07 -0700 Subject: [PATCH 14/14] feedback --- lib/vector-core/src/metrics/ddsketch.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/vector-core/src/metrics/ddsketch.rs b/lib/vector-core/src/metrics/ddsketch.rs index 69b32cf97e6f7..ee54fea3c1983 100644 --- a/lib/vector-core/src/metrics/ddsketch.rs +++ b/lib/vector-core/src/metrics/ddsketch.rs @@ -823,9 +823,11 @@ impl AgentDDSketch { // correct quantile behavior. match u32::try_from(*count) { Ok(c) => { - if c > 0 && *sum != 0.0 && sketch.count() > 0{ + let sc = sketch.count(); + + if c > 0 && *sum != 0.0 && sc > 0 { sketch.sum = *sum; - sketch.avg = *sum / f64::from(c); + sketch.avg = *sum / sc as f64; } } Err(_) => {