From 083cb26ba4bec7ea9ccbcb45589cbd23dd95d6b5 Mon Sep 17 00:00:00 2001 From: ddsign Date: Wed, 15 Jul 2026 15:43:31 -0400 Subject: [PATCH 1/2] fix: default capture-expression probes to snapshot rate (DEBUG-5730) When a log probe carries captureExpressions and no explicit sampling, default its sampling_snapshots_per_second to 1/s (snapshot rate) instead of the 5000/s log default. Evaluating capture expressions is cost-equivalent to a full snapshot; running them under the log rate limit spikes apm-processing CPU (the same class of issue that gated Java <1.64.0 and Python <4.11.0). --- datadog-live-debugger/src/parse_json.rs | 65 ++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/datadog-live-debugger/src/parse_json.rs b/datadog-live-debugger/src/parse_json.rs index 175805ce95..fbb9b4ca14 100644 --- a/datadog-live-debugger/src/parse_json.rs +++ b/datadog-live-debugger/src/parse_json.rs @@ -90,7 +90,25 @@ pub fn parse(json: &str) -> anyhow::Result { sampling_snapshots_per_second: parsed .sampling .map(|s| s.snapshots_per_second) - .unwrap_or(5000), + .unwrap_or_else(|| { + // DEBUG-5730: evaluating capture expressions is + // cost-equivalent to taking a full snapshot. When the + // probe carries no explicit sampling, rate-limit these + // probes like snapshots (1/s) instead of at the log + // default (5000/s), which drives an apm-processing CPU + // spike when the emitted snapshots run under the log + // rate limit. + let has_capture_expressions = parsed + .capture_expressions + .as_ref() + .map(|v| !v.is_empty()) + .unwrap_or(false); + if has_capture_expressions { + 1 + } else { + 5000 + } + }), capture_expressions: { let mut exprs = vec![]; for raw in parsed.capture_expressions.unwrap_or_default() { @@ -864,4 +882,49 @@ mod tests { unreachable!(); } } + + #[test] + fn test_capture_expressions_default_sampling_is_snapshot_rate() { + // DEBUG-5730: a log probe carrying capture expressions with no explicit + // sampling must default to the snapshot rate (1/s), not the log default + // (5000/s), which spikes apm-processing CPU. + let json = r#" +{ + "id": "3242910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 1, + "type": "LOG_PROBE", + "where": { + "typeName": "VetController", + "methodName": "showVetList" + }, + "template": "captured", + "segments": [{"str": "captured"}], + "captureExpressions": [ + { + "name": "foo", + "expr": {"dsl": "arg", "json": {"ref": "arg"}} + } + ] +} +"#; + + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + probe: + ProbeType::Log(LogProbe { + sampling_snapshots_per_second, + capture_expressions, + capture_snapshot, + .. + }), + .. + }) = parsed + { + assert_eq!(capture_expressions.len(), 1); + assert!(!capture_snapshot); + assert_eq!(sampling_snapshots_per_second, 1); + } else { + unreachable!(); + } + } } From c29a74e80717577d7de9c7f093839c3f0e09905d Mon Sep 17 00:00:00 2001 From: ddsign Date: Wed, 15 Jul 2026 16:04:53 -0400 Subject: [PATCH 2/2] apply the Ruby snapshot/log rate policy to sampling defaults Name the two defaults (SNAPSHOT_SAMPLING_SNAPSHOTS_PER_SECOND = 1, LOG_SAMPLING_SNAPSHOTS_PER_SECOND = 5000) and, absent explicit sampling, default any snapshot-emitting log probe (full captureSnapshot OR capture expressions) to the snapshot rate, matching the Ruby tracer's (@capture_snapshot || capture_expressions present) ? 1 : 5000 policy. Plain log probes keep the log rate. Also removes the duplicated 5000 literal on the service-config default. --- datadog-live-debugger/src/parse_json.rs | 106 ++++++++++++++++++++---- 1 file changed, 90 insertions(+), 16 deletions(-) diff --git a/datadog-live-debugger/src/parse_json.rs b/datadog-live-debugger/src/parse_json.rs index fbb9b4ca14..6147e3f496 100644 --- a/datadog-live-debugger/src/parse_json.rs +++ b/datadog-live-debugger/src/parse_json.rs @@ -15,6 +15,14 @@ use anyhow::Context; use serde::Deserialize; use std::fmt::{Display, Formatter}; +/// Default per-second sampling for log probes that emit a snapshot — either a +/// full `captureSnapshot` or capture expressions. Mirrors the tracer snapshot +/// rate; emitting these under the log rate limit spikes apm-processing CPU +/// (DEBUG-5730). +const SNAPSHOT_SAMPLING_SNAPSHOTS_PER_SECOND: u32 = 1; +/// Default per-second sampling for plain (non-snapshot) log probes. +const LOG_SAMPLING_SNAPSHOTS_PER_SECOND: u32 = 5000; + pub fn parse(json: &str) -> anyhow::Result { let parsed: RawTopLevelItem = serde_json::from_str(json)?; fn err(result: Result) -> anyhow::Result { @@ -29,7 +37,7 @@ pub fn parse(json: &str) -> anyhow::Result { sampling_snapshots_per_second: parsed .sampling .map(|s| s.snapshots_per_second) - .unwrap_or(5000), + .unwrap_or(LOG_SAMPLING_SNAPSHOTS_PER_SECOND), }) } probe_type => LiveDebuggingData::Probe({ @@ -91,22 +99,21 @@ pub fn parse(json: &str) -> anyhow::Result { .sampling .map(|s| s.snapshots_per_second) .unwrap_or_else(|| { - // DEBUG-5730: evaluating capture expressions is - // cost-equivalent to taking a full snapshot. When the - // probe carries no explicit sampling, rate-limit these - // probes like snapshots (1/s) instead of at the log - // default (5000/s), which drives an apm-processing CPU - // spike when the emitted snapshots run under the log - // rate limit. - let has_capture_expressions = parsed - .capture_expressions - .as_ref() - .map(|v| !v.is_empty()) - .unwrap_or(false); - if has_capture_expressions { - 1 + // Match the tracer rate policy: a probe that emits a + // snapshot (full snapshot or capture expressions) + // defaults to the snapshot rate; a plain log probe to + // the log rate. Absent explicit sampling, a snapshot at + // the log rate spikes apm-processing CPU (DEBUG-5730). + let emits_snapshot = parsed.capture_snapshot.unwrap_or(false) + || parsed + .capture_expressions + .as_ref() + .map(|v| !v.is_empty()) + .unwrap_or(false); + if emits_snapshot { + SNAPSHOT_SAMPLING_SNAPSHOTS_PER_SECOND } else { - 5000 + LOG_SAMPLING_SNAPSHOTS_PER_SECOND } }), capture_expressions: { @@ -927,4 +934,71 @@ mod tests { unreachable!(); } } + + #[test] + fn test_snapshot_probe_default_sampling_is_snapshot_rate() { + // A captureSnapshot probe with no explicit sampling defaults to the + // snapshot rate (1/s), matching the tracer policy. + let json = r#" +{ + "id": "4242910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 1, + "type": "LOG_PROBE", + "where": {"typeName": "X", "methodName": "y"}, + "template": "hi", + "segments": [{"str": "hi"}], + "captureSnapshot": true +} +"#; + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + probe: + ProbeType::Log(LogProbe { + sampling_snapshots_per_second, + capture_snapshot, + .. + }), + .. + }) = parsed + { + assert!(capture_snapshot); + assert_eq!(sampling_snapshots_per_second, 1); + } else { + unreachable!(); + } + } + + #[test] + fn test_plain_log_probe_default_sampling_is_log_rate() { + // A plain log probe (no snapshot, no capture expressions, no sampling) + // keeps the log rate default. + let json = r#" +{ + "id": "5242910d-d2ff-4679-85cc-bfc317d74e8f", + "version": 1, + "type": "LOG_PROBE", + "where": {"typeName": "X", "methodName": "y"}, + "template": "hi", + "segments": [{"str": "hi"}] +} +"#; + let parsed = parse_json(json).unwrap(); + if let LiveDebuggingData::Probe(Probe { + probe: + ProbeType::Log(LogProbe { + sampling_snapshots_per_second, + capture_snapshot, + capture_expressions, + .. + }), + .. + }) = parsed + { + assert!(!capture_snapshot); + assert!(capture_expressions.is_empty()); + assert_eq!(sampling_snapshots_per_second, 5000); + } else { + unreachable!(); + } + } }