From 561afd1674f88f53cb321af83e78963e1caaa5ad Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Fri, 24 Jul 2026 22:17:56 -0400 Subject: [PATCH] fix(sidecar): harden telemetry lifecycle state --- datadog-sidecar/src/entry.rs | 2 +- datadog-sidecar/src/service/session_info.rs | 24 +- datadog-sidecar/src/service/sidecar_server.rs | 3183 ++++++++++-- datadog-sidecar/src/service/stats_flusher.rs | 78 +- datadog-sidecar/src/service/telemetry.rs | 4385 +++++++++++++++-- 5 files changed, 6857 insertions(+), 815 deletions(-) diff --git a/datadog-sidecar/src/entry.rs b/datadog-sidecar/src/entry.rs index 1fd039dc1c..ee0d8c2cbc 100644 --- a/datadog-sidecar/src/entry.rs +++ b/datadog-sidecar/src/entry.rs @@ -113,7 +113,7 @@ where // Initialize telemetry sender synchronously before spawning the receiver task // This ensures the sender is available immediately, avoiding race conditions // where FFI calls might try to send telemetry before the receiver task starts - if let Some(rx) = init_telemetry_sender() { + if let Some(rx) = init_telemetry_sender(&server) { tokio::spawn(telemetry_action_receiver_task(server.clone(), rx)); } diff --git a/datadog-sidecar/src/service/session_info.rs b/datadog-sidecar/src/service/session_info.rs index a19bce2579..e0a383105c 100644 --- a/datadog-sidecar/src/service/session_info.rs +++ b/datadog-sidecar/src/service/session_info.rs @@ -98,12 +98,9 @@ impl SessionInfo { } /// Shuts down all running instances in the session. + #[cfg(test)] pub(crate) async fn shutdown_running_instances(&self) { - let runtimes: Vec = self - .lock_runtimes() - .drain() - .map(|(_, instance)| instance) - .collect(); + let runtimes = self.take_running_instances(); let instances_shutting_down: Vec<_> = runtimes .into_iter() @@ -113,11 +110,28 @@ impl SessionInfo { future::join_all(instances_shutting_down).await; } + #[cfg(test)] + pub(crate) fn take_running_instances(&self) -> Vec { + self.lock_runtimes() + .drain() + .map(|(_, instance)| instance) + .collect() + } + + pub(crate) fn take_runtime(&self, runtime_id: &str) -> Option { + self.lock_runtimes().remove(runtime_id) + } + + pub(crate) fn find_runtime(&self, runtime_id: &str) -> Option { + self.lock_runtimes().get(runtime_id).cloned() + } + /// Shuts down a specific runtime in the session. /// /// # Arguments /// /// * `runtime_id` - The ID of the runtime. + #[cfg(test)] pub(crate) async fn shutdown_runtime(&self, runtime_id: &str) { let maybe_runtime = { let mut runtimes = self.lock_runtimes(); diff --git a/datadog-sidecar/src/service/sidecar_server.rs b/datadog-sidecar/src/service/sidecar_server.rs index 9f2f46b797..5633a9bebb 100644 --- a/datadog-sidecar/src/service/sidecar_server.rs +++ b/datadog-sidecar/src/service/sidecar_server.rs @@ -5,7 +5,10 @@ use crate::log::{TemporarilyRetainedMapStats, MULTI_LOG_FILTER, MULTI_LOG_WRITER use crate::service::{ sidecar_interface::serve_sidecar_interface_connection, telemetry::{ - InitialTelemetryData, MetricsLogsClientSet, TelemetryCachedClient, TelemetryCachedClientSet, + ApplicationTelemetryDispatch, DirectTelemetryLifecycleRegistry, DirectTelemetryRetirement, + InitialTelemetryData, MetricsLogsClientSet, PendingApplicationAction, + TelemetryActionSender, TelemetryCachedClient, TelemetryCachedClientSet, + TelemetryWorkerMetadata, }, tracing::TraceFlusher, DynamicInstrumentationConfigState, InstanceId, QueueId, RuntimeInfo, RuntimeMetadata, @@ -24,12 +27,16 @@ use libdd_trace_utils::tracer_payload::TraceEncoding; use manual_future::ManualFutureCompleter; use std::borrow::Cow; use std::collections::hash_map::Entry; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, Weak}; use std::time::{Duration, SystemTime}; use tracing::{debug, error, info, trace, warn}; +#[cfg(test)] +use crate::service::telemetry::{telemetry_action_receiver_task, InternalTelemetryActions}; + use crate::config::get_product_endpoint; use crate::service::agent_info::AgentInfos; use crate::service::debugger_diagnostics_bookkeeper::{ @@ -100,10 +107,16 @@ pub struct SidecarServer { sessions: Arc>>, /// A `Mutex` guarded `HashMap` that keeps a count of each session. session_counter: Arc>>, + /// Number of live connections that have submitted work for each runtime. + instance_counter: Arc>>, /// A `Mutex` guarded `HashMap` that stores the active telemetry clients. pub(crate) telemetry_clients: TelemetryCachedClientSet, /// Telemetry clients for logs and metrics that are independent of application lifecycle. pub(crate) metrics_logs_clients: MetricsLogsClientSet, + /// Installed together with the direct-action receiver so lifecycle cleanup can be ordered + /// after every action already accepted by that receiver. + direct_telemetry_sender: Arc>>, + pub(crate) direct_telemetry_lifecycles: DirectTelemetryLifecycleRegistry, /// A `Mutex` guarded optional `ManualFutureCompleter` for telemetry configuration. pub self_telemetry_config: Arc>>>, @@ -156,11 +169,37 @@ impl ConnectionSidecarHandler { } fn track_instance(&self, instance_id: &InstanceId) { - self.instances.lock_or_panic().insert(instance_id.clone()); + if self.instances.lock_or_panic().insert(instance_id.clone()) { + *self + .server + .instance_counter + .lock_or_panic() + .entry(instance_id.clone()) + .or_default() += 1; + } + } + + fn release_instances(&self) -> Vec { + let instances = self.instances.lock_or_panic().drain().collect::>(); + let mut counters = self.server.instance_counter.lock_or_panic(); + instances + .into_iter() + .filter(|instance_id| match counters.entry(instance_id.clone()) { + Entry::Occupied(mut entry) if *entry.get() > 1 => { + *entry.get_mut() -= 1; + false + } + Entry::Occupied(entry) => { + entry.remove(); + true + } + Entry::Vacant(_) => false, + }) + .collect() } async fn cleanup(&self) { - let instances: Vec = self.instances.lock_or_panic().iter().cloned().collect(); + let mut instances = self.release_instances(); if let Some(session_id) = self.session_id.get() { let stop = { @@ -178,19 +217,12 @@ impl ConnectionSidecarHandler { }; if stop { self.server.stop_session(session_id).await; + instances.retain(|instance_id| instance_id.session_id != *session_id); } } for instance_id in instances { - let maybe_session = self - .server - .sessions - .lock_or_panic() - .get(&instance_id.session_id) - .cloned(); - if let Some(session) = maybe_session { - session.shutdown_runtime(&instance_id.runtime_id).await; - } + self.server.stop_runtime(&instance_id).await; } } } @@ -241,15 +273,89 @@ impl SidecarServer { } } - fn get_runtime(&self, instance_id: &InstanceId) -> RuntimeInfo { - let session = self.get_session(&instance_id.session_id); - session.get_runtime(&instance_id.runtime_id) + pub(crate) fn find_session(&self, session_id: &str) -> Option { + self.sessions.lock_or_panic().get(session_id).cloned() + } + + pub(crate) fn get_runtime(&self, instance_id: &InstanceId) -> RuntimeInfo { + self.direct_telemetry_lifecycles + .activate_with(instance_id, || { + let session = self.get_session(&instance_id.session_id); + session + .find_runtime(&instance_id.runtime_id) + .unwrap_or_else(|| session.get_runtime(&instance_id.runtime_id)) + }) + } + + pub(crate) fn find_runtime(&self, instance_id: &InstanceId) -> Option { + self.find_session(&instance_id.session_id)? + .find_runtime(&instance_id.runtime_id) + } + + pub(crate) fn install_direct_telemetry_sender(&self, sender: TelemetryActionSender) { + self.direct_telemetry_sender.lock_or_panic().replace(sender); + } + + async fn retire_direct_telemetry(&self, scope: DirectTelemetryRetirement) { + let sender = self.direct_telemetry_sender.lock_or_panic().clone(); + if let Some(sender) = sender { + match sender.retire(scope.clone()).await { + Ok(()) => return, + Err(error) => { + warn!( + "Failed to order direct telemetry cleanup through receiver; \ + falling back to synchronous cleanup: {error}" + ); + } + } + } + + match scope { + DirectTelemetryRetirement::Runtimes(instances) => { + let retired = self.direct_telemetry_lifecycles.retire_runtimes(&instances); + self.metrics_logs_clients.remove_runtimes(&retired); + } + DirectTelemetryRetirement::Session(session_id) => { + self.direct_telemetry_lifecycles.retire_session(&session_id); + self.metrics_logs_clients.remove_session(&session_id); + } + } + } + + async fn stop_runtime(&self, instance_id: &InstanceId) { + let instances = HashSet::from([instance_id.clone()]); + let retirements = self + .direct_telemetry_lifecycles + .begin_retire_runtimes(&instances); + self.retire_direct_telemetry(DirectTelemetryRetirement::Runtimes(retirements.clone())) + .await; + let runtime = + self.direct_telemetry_lifecycles + .finish_retire_runtimes(&retirements, |ready| { + ready.contains(instance_id).then(|| { + self.find_session(&instance_id.session_id) + .and_then(|session| session.take_runtime(&instance_id.runtime_id)) + }) + }); + if let Some(Some(runtime)) = runtime { + runtime.shutdown().await; + } } async fn stop_session(&self, session_id: &str) { - let session = match self.sessions.lock_or_panic().remove(session_id) { - Some(session) => session, - None => return, + self.direct_telemetry_lifecycles + .begin_retire_session(session_id); + self.telemetry_clients.remove_pending_session(session_id); + self.retire_direct_telemetry(DirectTelemetryRetirement::Session(session_id.to_string())) + .await; + let session = self + .direct_telemetry_lifecycles + .finish_retire_session(session_id, || { + self.sessions.lock_or_panic().remove(session_id) + }) + .flatten(); + let Some(session) = session else { + return; }; info!("Shutting down session: {}", session_id); @@ -324,35 +430,45 @@ impl SidecarServer { } pub async fn compute_stats(&self) -> SidecarStats { - let (futures, metric_count, active_telemetry_clients) = { - let application_clients = self.telemetry_clients.clients(); - let metrics_logs_clients = self.metrics_logs_clients.clients(); - let mut futures = - Vec::with_capacity(application_clients.len() + metrics_logs_clients.len()); - let mut metric_count = 0; - for client in application_clients.iter() { - if let Some(client) = client.lock_or_panic().as_ref() { - metric_count += client.telemetry_metrics.len() as u32; - if let Ok(stats) = client.worker.stats() { - futures.push(stats); - } - } - } - for client in metrics_logs_clients.iter() { - let client = client.lock_or_panic(); - metric_count += client.telemetry_metrics.len() as u32; - if let Ok(stats) = client.worker.stats() { - futures.push(stats); - } + let application_clients = self.telemetry_clients.clients(); + let active_telemetry_clients = application_clients + .iter() + .filter(|client| client.lock_or_panic().as_ref().is_some()) + .count() as u32; + let cached_clients = application_clients + .into_iter() + .chain(self.metrics_logs_clients.clients()) + .collect::>(); + let mut workers = Vec::with_capacity(cached_clients.len()); + let mut telemetry_metrics_contexts = 0; + for client in cached_clients { + if let Some(client) = client.lock_or_panic().as_ref() { + workers.push(client.worker.clone()); + telemetry_metrics_contexts += client.telemetry_metrics.len() as u32; } + } + workers.extend( + self.span_concentrators + .lock_or_panic() + .values() + .filter_map(|state| state.telemetry.clone()), + ); - let active_telemetry_clients = - (application_clients.len() + metrics_logs_clients.len()) as u32; - (futures, metric_count, active_telemetry_clients) - }; - + let mut telemetry_stats_errors = 0; + let futures = workers + .into_iter() + .filter_map(|worker| match worker.stats() { + Ok(stats) => Some(stats), + Err(_) => { + telemetry_stats_errors += 1; + None + } + }); let telemetry_stats = futures::future::join_all(futures).await; - let telemetry_stats_errors = telemetry_stats.iter().filter(|r| r.is_err()).count() as u32; + telemetry_stats_errors += telemetry_stats + .iter() + .filter(|result| result.is_err()) + .count() as u32; let sessions = self.sessions.lock_or_panic(); SidecarStats { @@ -389,9 +505,8 @@ impl SidecarServer { .sum(), remote_configs: self.remote_configs.stats(), debugger_diagnostics_bookkeeping: self.debugger_diagnostics_bookkeeper.stats(), - telemetry_metrics_contexts: metric_count, - telemetry_worker_errors: telemetry_stats_errors - + telemetry_stats.iter().filter(|v| v.is_err()).count() as u32, + telemetry_metrics_contexts, + telemetry_worker_errors: telemetry_stats_errors, telemetry_worker: telemetry_stats.into_iter().filter_map(|v| v.ok()).sum(), log_filter: MULTI_LOG_FILTER.stats(), log_writer: MULTI_LOG_WRITER.stats(), @@ -403,6 +518,199 @@ impl SidecarServer { } } +#[derive(Debug)] +struct ScheduledApplicationActions { + remove_client: bool, + next_lifecycle_actions: Vec, + terminal_handle: Option>, +} + +async fn await_terminal_handoff(handle: tokio::task::JoinHandle<()>) { + if let Err(error) = handle.await { + warn!("Terminal telemetry handoff failed: {error}"); + } +} + +fn remove_application_client_after_handoff( + clients: TelemetryCachedClientSet, + service: String, + env: String, + client: Arc>>, + terminal_handle: Option>, +) { + if let Some(handle) = terminal_handle { + tokio::spawn(async move { + await_terminal_handoff(handle).await; + clients.remove_telemetry_client(&service, &env, &client); + }); + } else { + clients.remove_telemetry_client(&service, &env, &client); + } +} + +enum ScheduledApplicationWork { + Direct(Vec), + Composer(Vec), +} + +fn push_direct_application_work( + work: &mut Vec, + action: TelemetryActions, +) { + match work.last_mut() { + Some(ScheduledApplicationWork::Direct(existing)) => existing.push(action), + _ => work.push(ScheduledApplicationWork::Direct(vec![action])), + } +} + +fn schedule_application_actions( + telemetry_mutex: &Arc>>, + mut actions: Vec, + created: bool, + instance_id: &InstanceId, + queue_id: QueueId, +) -> Result> { + let mut telemetry_guard = telemetry_mutex.lock_or_panic(); + let Some(telemetry) = telemetry_guard.as_mut() else { + warn!("enqueue_actions: telemetry client unavailable for instance {instance_id:?}"); + return Err(actions); + }; + if telemetry.is_stopping() { + warn!("enqueue_actions: telemetry client stopping for instance {instance_id:?}"); + return Err(actions); + } + + let next_lifecycle_actions = PendingApplicationAction::split_after_first_stop(&mut actions); + + for pending_action in &actions { + if let SidecarAction::AddTelemetryMetricPoint((name, _, _)) = &pending_action.action { + if !telemetry.telemetry_metrics.contains_key(name) { + if let Some(metric) = &pending_action.metric_registration { + debug!( + "Registering pending telemetry metric {name} from instance {:?}", + pending_action.origin + ); + telemetry.register_metric(metric.clone()); + } + } + } + } + + let mut work = Vec::new(); + let mut buffered_info_changed = false; + let mut remove_client = false; + + for pending_action in actions { + let action = pending_action.action; + if created && InitialTelemetryData::contains_seeded_action(&action) { + continue; + } + match action { + SidecarAction::Telemetry(TelemetryActions::AddIntegration(ref integration)) => { + if telemetry.shared.integrations.insert(integration.clone()) { + if let Some(action) = telemetry.process_action(action) { + push_direct_application_work(&mut work, action); + } + buffered_info_changed = true; + } + } + SidecarAction::PhpComposerTelemetryFile(path) => { + if telemetry.shared.composer_paths.insert(path.clone()) { + work.push(ScheduledApplicationWork::Composer(vec![path])); + buffered_info_changed = true; + } + } + SidecarAction::Telemetry(TelemetryActions::AddConfig(_)) => { + telemetry.shared.config_sent = true; + buffered_info_changed = true; + if let Some(action) = telemetry.process_action(action) { + push_direct_application_work(&mut work, action); + } + } + SidecarAction::Telemetry(TelemetryActions::AddEndpoint(_)) => { + telemetry.shared.last_endpoints_push = SystemTime::now(); + buffered_info_changed = true; + if let Some(action) = telemetry.process_action(action) { + push_direct_application_work(&mut work, action); + } + } + SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)) => { + telemetry.mark_stopping(); + remove_client = true; + if let Some(action) = telemetry.process_action(action) { + push_direct_application_work(&mut work, action); + } + } + _ => { + if let Some(action) = telemetry.process_action(action) { + push_direct_application_work(&mut work, action); + } + } + } + } + + if buffered_info_changed { + info!( + "Buffered telemetry info changed for instance {instance_id:?} and queue_id {queue_id:?}" + ); + telemetry.write_shm_file(); + } else if !remove_client { + telemetry.retry_shm_file_if_due(); + } + + let (terminal_completion, terminal_worker_join) = if remove_client { + let (completion, receiver) = tokio::sync::watch::channel(false); + telemetry.terminal_handoff = Some(receiver); + (Some(completion), telemetry.worker_join.take()) + } else { + (None, None) + }; + let mut terminal_handle = None; + if !work.is_empty() { + let worker = telemetry.worker.clone(); + let last_handle = telemetry.handle.take(); + let handle = tokio::spawn(async move { + if let Some(last_handle) = last_handle { + last_handle.await.ok(); + }; + for work_item in work { + let processed = match work_item { + ScheduledApplicationWork::Direct(actions) => actions, + ScheduledApplicationWork::Composer(paths) => { + TelemetryCachedClient::process_composer_paths(paths).await + } + }; + if !processed.is_empty() { + debug!("Sending processed application actions: {processed:?}"); + if let Err(error) = worker.send_msgs(processed).await { + warn!("Failed to send application telemetry actions: {error}"); + } + } + } + drop(worker); + if let Some(worker_join) = terminal_worker_join { + if let Err(error) = worker_join.await { + warn!("Application telemetry worker shutdown failed: {error}"); + } + } + if let Some(completion) = terminal_completion { + completion.send_replace(true); + } + }); + if remove_client { + terminal_handle = Some(handle); + } else if let Some(telemetry) = telemetry_guard.as_mut() { + telemetry.handle = Some(handle); + } + } + + Ok(ScheduledApplicationActions { + remove_client, + next_lifecycle_actions, + terminal_handle, + }) +} + impl SidecarInterface for ConnectionSidecarHandler { fn recv_counter(&self) -> &AtomicU64 { &self.submitted_payloads @@ -426,19 +734,24 @@ impl SidecarInterface for ConnectionSidecarHandler { self.track_instance(&instance_id); let connection_metric_registrations = self.metric_registrations.lock_or_panic().clone(); let session = self.server.get_session(&instance_id.session_id); - let trace_config = session.get_trace_config(); - let runtime_metadata = RuntimeMetadata::new( - trace_config.language.clone(), - trace_config.language_version.clone(), - trace_config.tracer_version.clone(), - ); + let (runtime_metadata, trace_endpoint) = { + let trace_config = session.get_trace_config(); + ( + RuntimeMetadata::new( + trace_config.language.clone(), + trace_config.language_version.clone(), + trace_config.tracer_version.clone(), + ), + trace_config.endpoint.clone(), + ) + }; let ffe_http_client = self.server.ffe_http_client.clone(); let actions: Vec = actions .into_iter() .filter(|a| match a { SidecarAction::FfeExposureBatch(batch) => { - if let Some(base) = trace_config.endpoint.as_ref() { + if let Some(base) = trace_endpoint.as_ref() { if let Some(ep) = ffe_exposures_flusher::exposure_endpoint(base) { let batch = batch.clone(); let client = ffe_http_client.clone(); @@ -483,183 +796,172 @@ impl SidecarInterface for ConnectionSidecarHandler { return; } - let rt_info = self.server.get_runtime(&instance_id); - let mut applications = rt_info.lock_applications(); + let (service, env) = { + let rt_info = self.server.get_runtime(&instance_id); + let applications = rt_info.lock_applications(); + let Some(application) = applications.get(&queue_id) else { + info!( + "No application found for instance {instance_id:?} and queue_id {queue_id:?}" + ); + return; + }; + ( + application + .service_name + .clone() + .unwrap_or_else(|| "unknown-service".to_string()), + application + .env + .clone() + .unwrap_or_else(|| "none".to_string()), + ) + }; - if let Entry::Occupied(entry) = applications.entry(queue_id) { - let service = entry - .get() - .service_name - .as_deref() - .unwrap_or("unknown-service"); - let env = entry.get().env.as_deref().unwrap_or("none"); - - let process_tags = session.process_tags_with_svc_source(); - // Pre-compute session config so both the primary and retry get_or_create calls - // can use it without re-locking the session. - let session_config = session - .session_config - .lock_or_panic() - .as_ref() - .cloned() - .unwrap_or_else(|| { - warn!("Failed to get telemetry session config for {instance_id:?}"); - Config::default() - }); + let process_tags = session.process_tags_with_svc_source(); + // Pre-compute session config so replacement get_or_create calls can use it + // without re-locking the session. + let session_config = session + .session_config + .lock_or_panic() + .as_ref() + .cloned() + .unwrap_or_else(|| { + warn!("Failed to get telemetry session config for {instance_id:?}"); + Config::default() + }); - // Get or create the telemetry client. If we observe None under the lock it means - // another thread called take() (Stop) in the narrow window between get_or_create - // returning and us acquiring the lock — retry once to get a fresh client. - let (telemetry_mutex, created) = - self.server.telemetry_clients.get_or_create_with_initial( - service, - env, + let mut pending_actions = PendingApplicationAction::from_actions( + &instance_id, + actions, + &connection_metric_registrations, + ); + loop { + let mut initialized_terminal_handle = None; + let dispatch = self.server.telemetry_clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new( + &service, + &env, &instance_id, &runtime_metadata, - || session_config.clone(), process_tags.clone(), - || InitialTelemetryData::from_actions(&actions), - ); - let (telemetry_mutex, created) = if telemetry_mutex.lock_or_panic().is_none() { - self.server.telemetry_clients.get_or_create_with_initial( - service, - env, + ), + pending_actions, + || session_config.clone(), + |client, actions| match schedule_application_actions( + client, + actions, + true, &instance_id, - &runtime_metadata, - || session_config, - process_tags, - || InitialTelemetryData::from_actions(&actions), - ) - } else { - (telemetry_mutex, created) - }; - let mut telemetry_guard = telemetry_mutex.lock_or_panic(); - let Some(telemetry) = telemetry_guard.as_mut() else { - // Extremely rare: the client was stopped between the two get_or_create calls. - warn!("enqueue_actions: telemetry client stopped during retry for instance {instance_id:?}; dropping actions"); - return; - }; - - // Auto-register any metrics known to this connection but not yet registered - // in this telemetry client (e.g., the client was just created for a new service/env). - for action in &actions { - if let SidecarAction::AddTelemetryMetricPoint((name, _, _)) = action { - if !telemetry.telemetry_metrics.contains_key(name) { - if let Some(metric) = connection_metric_registrations.get(name) { - telemetry.register_metric(metric.clone()); + queue_id, + ) { + Ok(scheduled) => { + initialized_terminal_handle = scheduled.terminal_handle; + scheduled.remove_client + } + Err(returned) => { + warn!( + "New telemetry lifecycle rejected {} initialization actions", + returned.len() + ); + false + } + }, + ); + let (telemetry_mutex, actions, created, initialized_terminal) = match dispatch { + ApplicationTelemetryDispatch::Pending => return, + ApplicationTelemetryDispatch::Handoff { + mut completion, + actions, + } => { + while !*completion.borrow() { + if completion.changed().await.is_err() { + warn!( + "Application telemetry handoff ended without completion for \ + {service:?}/{env:?}" + ); + break; } } + pending_actions = actions; + continue; + } + ApplicationTelemetryDispatch::Ready { + client, + actions, + created, + remove_client, + } => (client, actions, created, remove_client), + }; + + if initialized_terminal { + info!("Removing terminal telemetry client for instance {instance_id:?}"); + if actions.is_empty() { + remove_application_client_after_handoff( + self.server.telemetry_clients.clone(), + service, + env, + telemetry_mutex, + initialized_terminal_handle, + ); + return; } + if let Some(handle) = initialized_terminal_handle { + await_terminal_handoff(handle).await; + } + self.server.telemetry_clients.remove_telemetry_client( + &service, + &env, + &telemetry_mutex, + ); + pending_actions = actions; + continue; + } + if actions.is_empty() { + return; } - let mut actions_to_process: Vec = vec![]; - let mut composer_paths_to_process = vec![]; - let mut buffered_info_changed = false; - let mut remove_client = false; - - for action in actions { - match action { - SidecarAction::Telemetry(TelemetryActions::AddIntegration(ref integration)) => { - if telemetry.shared.integrations.insert(integration.clone()) { - if !created { - actions_to_process.push(action); - } - buffered_info_changed = true; - } - } - SidecarAction::PhpComposerTelemetryFile(path) => { - if telemetry.shared.composer_paths.insert(path.clone()) { - composer_paths_to_process.push(path); - buffered_info_changed = true; - } + match schedule_application_actions( + &telemetry_mutex, + actions, + created, + &instance_id, + queue_id, + ) { + Ok(scheduled) => { + if scheduled.remove_client { + info!("Removing telemetry client for instance {instance_id:?}"); } - SidecarAction::Telemetry(TelemetryActions::AddConfig(_)) => { - telemetry.shared.config_sent = true; - buffered_info_changed = true; - if !created { - actions_to_process.push(action); + if scheduled.next_lifecycle_actions.is_empty() { + if scheduled.remove_client { + remove_application_client_after_handoff( + self.server.telemetry_clients.clone(), + service, + env, + telemetry_mutex, + scheduled.terminal_handle, + ); } + return; } - SidecarAction::Telemetry(TelemetryActions::AddDependency(_)) if created => {} - SidecarAction::Telemetry(TelemetryActions::AddEndpoint(_)) => { - telemetry.shared.last_endpoints_push = SystemTime::now(); - buffered_info_changed = true; - actions_to_process.push(action); + if let Some(handle) = scheduled.terminal_handle { + await_terminal_handoff(handle).await; } - SidecarAction::Telemetry(TelemetryActions::Lifecycle( - LifecycleAction::Stop, - )) => { - remove_client = true; - actions_to_process.push(action); - } - _ => { - actions_to_process.push(action); + if scheduled.remove_client { + self.server.telemetry_clients.remove_telemetry_client( + &service, + &env, + &telemetry_mutex, + ); } + pending_actions = scheduled.next_lifecycle_actions; + } + Err(returned) => { + // The selected client became unavailable or terminal before this batch + // acquired its mutex. Ownership remains here and is atomically offered to + // the next cache lifecycle. + pending_actions = returned; } } - - if buffered_info_changed { - info!( - "Buffered telemetry info changed for instance {instance_id:?} and queue_id {queue_id:?}" - ); - telemetry.write_shm_file(); - } - - // take() must happen INSIDE the spawned task, after process_actions completes, - // so that a Config batch spawned before a Stop batch still finds Some when it - // runs (the last_handle chain guarantees Stop runs after Config). - let do_take = remove_client; - - if !actions_to_process.is_empty() { - let telemetry_mutex_clone = telemetry_mutex.clone(); - let worker = telemetry.worker.clone(); - let last_handle = telemetry.handle.take(); - telemetry.handle = Some(tokio::spawn(async move { - if let Some(last_handle) = last_handle { - last_handle.await.ok(); - }; - let processed = { - let mut guard = telemetry_mutex_clone.lock_or_panic(); - let processed = guard - .as_mut() - .map(|t| t.process_actions(actions_to_process)) - .unwrap_or_default(); - if do_take { - guard.take(); // drop client after Stop action is processed - } - processed - }; - debug!("Sending Processed Actions :{processed:?}"); - worker.send_msgs(processed).await.ok(); - })); - } - - if !composer_paths_to_process.is_empty() { - let worker = telemetry.worker.clone(); - let last_handle = telemetry.handle.take(); - telemetry.handle = Some(tokio::spawn(async move { - if let Some(last_handle) = last_handle { - last_handle.await.ok(); - }; - let composer_actions = - TelemetryCachedClient::process_composer_paths(composer_paths_to_process) - .await; - debug!("Sending Composer Paths :{composer_actions:?}"); - worker.send_msgs(composer_actions).await.ok(); - })); - } - - // telemetry borrow ends after the last use of telemetry.handle above. - // Remove from the map synchronously so new get_or_create calls get a fresh entry; - // take() is deferred to the spawned task to avoid racing with in-flight tasks. - if remove_client { - self.server - .telemetry_clients - .remove_telemetry_client(service, env); - info!("Removing telemetry client for instance {instance_id:?}"); - } - } else { - info!("No application found for instance {instance_id:?} and queue_id {queue_id:?}"); } } @@ -700,6 +1002,35 @@ impl SidecarInterface for ConnectionSidecarHandler { debug!("Set session config for {session_id} to {config:?}"); let session = self.server.get_session(&session_id); + if !is_fork { + let instance_ids = session + .lock_runtimes() + .values() + .map(|runtime| runtime.instance_id.clone()) + .collect::>(); + let retirements = self + .server + .direct_telemetry_lifecycles + .begin_retire_runtimes(&instance_ids); + self.server + .retire_direct_telemetry(DirectTelemetryRetirement::Runtimes(retirements.clone())) + .await; + let runtimes = self + .server + .direct_telemetry_lifecycles + .finish_retire_runtimes(&retirements, |ready| { + ready + .iter() + .filter_map(|instance_id| session.take_runtime(&instance_id.runtime_id)) + .collect::>() + }); + futures::future::join_all( + runtimes + .into_iter() + .map(|runtime| async move { runtime.shutdown().await }), + ) + .await; + } session .pid .store(self.connection.peer().pid as i32, Ordering::Relaxed); @@ -830,20 +1161,14 @@ impl SidecarInterface for ConnectionSidecarHandler { )); if let Some(completer) = self.server.self_telemetry_config.lock_or_panic().take() { - #[allow(clippy::expect_used)] - let config = session - .session_config - .lock_or_panic() - .as_ref() - .expect("Expected session_config to be Some(Config) but received None") - .clone(); - tokio::spawn(async move { - completer.complete(config).await; - }); - } - - if !is_fork { - session.shutdown_running_instances().await; + let config = session.session_config.lock_or_panic().as_ref().cloned(); + if let Some(config) = config { + tokio::spawn(async move { + completer.complete(config).await; + }); + } else { + warn!("Session telemetry config unexpectedly unavailable for {session_id}"); + } } } @@ -881,14 +1206,12 @@ impl SidecarInterface for ConnectionSidecarHandler { } async fn shutdown_runtime(&self, instance_id: InstanceId) { - let session = self.server.get_session(&instance_id.session_id); - tokio::spawn(async move { session.shutdown_runtime(&instance_id.runtime_id).await }); + self.server.stop_runtime(&instance_id).await; } async fn shutdown_session(&self) { - let server = self.server.clone(); let session_id = self.session_id.get().cloned().unwrap_or_default(); - tokio::spawn(async move { server.stop_session(&session_id).await }); + self.server.stop_session(&session_id).await; } async fn send_trace_v04_shm( @@ -1021,7 +1344,7 @@ impl SidecarInterface for ConnectionSidecarHandler { debug!("Registered remote config metadata: instance {instance_id:?}, queue_id: {queue_id:?}, service: {service_name}, env: {env_name}, version: {app_version}"); let session = self.server.get_session(&instance_id.session_id); - let runtime_info = session.get_runtime(&instance_id.runtime_id); + let runtime_info = self.server.get_runtime(&instance_id); let mut applications = runtime_info.lock_applications(); let app = applications.entry(queue_id).or_default(); app.set_metadata(env_name, app_version, service_name, global_tags); @@ -1046,7 +1369,7 @@ impl SidecarInterface for ConnectionSidecarHandler { ) { self.track_instance(&instance_id); let session = self.server.get_session(&instance_id.session_id); - let runtime_info = session.get_runtime(&instance_id.runtime_id); + let runtime_info = self.server.get_runtime(&instance_id); let mut applications = runtime_info.lock_applications(); let app = applications.entry(queue_id).or_default(); let Some(notify_target) = self.server.get_notify_target(&session) else { @@ -1087,14 +1410,9 @@ impl SidecarInterface for ConnectionSidecarHandler { let session_id = self.session_id.get().map(|s| s.as_str()).unwrap_or(""); let session = self.server.get_session(session_id); // Lazily create the concentrator on first IPC span for this (env, version, service). - if let Some(state) = get_or_create_concentrator( - &self.server.span_concentrators, - &self.server.metrics_logs_clients, - &env, - &version, - session_id, - &session, - ) { + if let Some(state) = + get_or_create_concentrator(&self.server.span_concentrators, &env, &version, &session) + { let mut peer_tag_buf = Vec::new(); let input = span.as_shm_input(&mut peer_tag_buf); state.concentrator.add_span(&input); @@ -1107,23 +1425,25 @@ impl SidecarInterface for ConnectionSidecarHandler { if let Err(e) = tokio::spawn(async move { flusher.flush().await }).await { error!("Failed flushing traces: {e:?}"); } - flush_all_stats_now(&self.server.span_concentrators).await; + let stats_states = { + let concentrators = self.server.span_concentrators.lock_or_panic(); + concentrators.values().cloned().collect::>() + }; + flush_all_stats_now(&stats_states).await; debug!("Finished executing flush() for traces and stats") } if options.telemetry { - let workers = self - .server - .telemetry_clients - .clients() - .into_iter() - .filter_map(|client| { - client - .lock_or_panic() - .as_ref() - .map(|client| client.worker.clone()) - }) - .chain(self.server.metrics_logs_clients.workers()) - .collect::>(); + let mut workers = self.server.telemetry_clients.workers(); + workers.extend(self.server.metrics_logs_clients.workers()); + let stats_states = { + let concentrators = self.server.span_concentrators.lock_or_panic(); + concentrators.values().cloned().collect::>() + }; + workers.extend( + stats_states + .into_iter() + .filter_map(|state| state.telemetry.clone()), + ); futures::future::join_all(workers.into_iter().map(|worker| async move { let _ = worker .send_msg(TelemetryActions::Lifecycle( @@ -1193,9 +1513,15 @@ impl SidecarInterface for ConnectionSidecarHandler { mod tests { use super::*; use crate::service::{FfeEvaluationMetric, FfeExposure, FfeExposureBatch, FfeTelemetryContext}; + use datadog_ipc::shm_stats::ShmSpanInput; use httpmock::{Method::POST, MockServer}; - use tokio::sync::Barrier; - use tokio::time::{sleep, Duration as TokioDuration}; + use libdd_trace_stats::span_concentrator::FixedAggregationKey; + use libdd_trace_utils::test_utils::create_send_data; + use std::path::PathBuf; + use std::time::Instant; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::sync::{oneshot, Barrier}; + use tokio::time::{sleep, timeout, Duration as TokioDuration}; /// Build a handler backed by a throwaway socketpair connection. These tests exercise /// `enqueue_actions`, which uses only the shared server state and never reads the connection, @@ -1207,46 +1533,904 @@ mod tests { ConnectionSidecarHandler::new(server, conn) } - fn ffe_context() -> FfeTelemetryContext { - FfeTelemetryContext { - service: "svc".to_owned(), - env: "prod".to_owned(), - version: "1".to_owned(), + fn test_session_config(endpoint: Endpoint, root_service: &str) -> SessionConfig { + SessionConfig { + dogstatsd_endpoint: endpoint.clone(), + endpoint, + language: "php".to_string(), + language_version: "8.3".to_string(), + tracer_version: "test".to_string(), + flush_interval: Duration::from_secs(60), + remote_config_poll_interval: Duration::from_secs(60), + telemetry_heartbeat_interval: Duration::from_secs(60), + telemetry_extended_heartbeat_interval: Duration::from_secs(3600), + force_flush_size: 0, + force_drop_size: 0, + retry_interval: Duration::from_millis(10), + log_level: "off".to_string(), + log_file: crate::config::LogMethod::Disabled, + remote_config_products: Vec::new(), + remote_config_capabilities: Vec::new(), + remote_config_enabled: false, + process_tags: Vec::new(), + peer_tag_keys: Vec::new(), + span_kinds_stats_computed: Vec::new(), + hostname: String::new(), + root_service: root_service.to_string(), + root_session_id: None, + parent_session_id: None, + otlp_metrics_endpoint: None, } } - fn ffe_exposure(subject_id: &str) -> FfeExposure { - FfeExposure { - timestamp_ms: 123, - flag_key: "flag".to_owned(), - subject_id: subject_id.to_owned(), - subject_attributes_json: "{}".to_owned(), - allocation_key: "alloc".to_owned(), - variant: "variant".to_owned(), - } - } + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn shutdown_removes_runtime_owned_telemetry() { + const SERVICE: &str = "cleanup-service"; + const ENV: &str = "test"; + const METRIC: &str = "cleanup.metric"; - fn ffe_metric() -> FfeEvaluationMetric { - FfeEvaluationMetric { - flag_key: "flag".to_owned(), - variant: "variant".to_owned(), - reason: "TARGETING_MATCH".to_owned(), - error_type: None, - allocation_key: Some("alloc".to_owned()), + let server = SidecarServer::default(); + let handler = test_handler(server.clone()); + handler + .session_id + .set("session".to_string()) + .expect("test handler session should be unset"); + let runtime_a = InstanceId::new("session", "runtime-a"); + let runtime_b = InstanceId::new("session", "runtime-b"); + let other_runtime = InstanceId::new("other-session", "runtime"); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + + for instance_id in [&runtime_a, &runtime_b, &other_runtime] { + server.metrics_logs_clients.get_or_create_metrics_logs( + SERVICE, + ENV, + instance_id, + &runtime_metadata, + Config::default, + Vec::new(), + ); } + assert!(server.metrics_logs_clients.register_metric( + &runtime_a, + SERVICE, + ENV, + MetricContext { + name: METRIC.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + )); + assert!(server.metrics_logs_clients.register_metric( + &other_runtime, + SERVICE, + ENV, + MetricContext { + name: "other.cleanup.metric".to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + )); + handler.shutdown_runtime(runtime_a.clone()).await; + timeout(TokioDuration::from_secs(1), async { + while server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_a, SERVICE, ENV) + .is_some() + { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("runtime telemetry cleanup should complete"); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_a, SERVICE, ENV) + .is_none()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_b, SERVICE, ENV) + .is_some()); + server.stop_runtime(&runtime_a).await; + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_a, SERVICE, ENV) + .is_none()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_b, SERVICE, ENV) + .is_some()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&other_runtime, SERVICE, ENV) + .is_some()); + assert!(!server + .metrics_logs_clients + .registered_metrics(&other_runtime, SERVICE, ENV) + .is_empty()); + + handler.shutdown_session().await; + timeout(TokioDuration::from_secs(1), async { + while server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_b, SERVICE, ENV) + .is_some() + || !server + .metrics_logs_clients + .registered_metrics(&runtime_b, SERVICE, ENV) + .is_empty() + { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("session telemetry cleanup should complete"); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_b, SERVICE, ENV) + .is_none()); + assert!(server + .metrics_logs_clients + .registered_metrics(&runtime_b, SERVICE, ENV) + .is_empty()); + assert!(server.sessions.lock_or_panic().get("session").is_none()); + server.stop_session("session").await; + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&runtime_b, SERVICE, ENV) + .is_none()); + assert!(server + .metrics_logs_clients + .registered_metrics(&runtime_b, SERVICE, ENV) + .is_empty()); + assert!(server.sessions.lock_or_panic().get("session").is_none()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&other_runtime, SERVICE, ENV) + .is_some()); + assert!(!server + .metrics_logs_clients + .registered_metrics(&other_runtime, SERVICE, ENV) + .is_empty()); } #[tokio::test] #[cfg_attr(miri, ignore)] - async fn ffe_exposure_actions_dispatch_without_registered_application() { - let http_server = MockServer::start_async().await; - let exposures_mock = http_server - .mock_async(|when, then| { - when.method(POST) - .path(ffe_exposures_flusher::EVP_EXPOSURES_PATH); - then.status(202); - }) - .await; + async fn connection_cleanup_retires_only_the_last_runtime_owner() { + const SERVICE: &str = "shared-runtime"; + const ENV: &str = "test"; + + let server = SidecarServer::default(); + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + server.install_direct_telemetry_sender(sender.clone()); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(server.clone(), receiver)); + let handler_a = test_handler(server.clone()); + let handler_b = test_handler(server.clone()); + let instance = InstanceId::new("session", "runtime"); + let session = server.get_session(&instance.session_id); + server.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + handler_a.track_instance(&instance); + handler_b.track_instance(&instance); + + let direct_log = |message: &str| InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::AddLog(( + libdd_telemetry::worker::LogIdentifier { identifier: 1 }, + libdd_telemetry::data::Log { + message: message.to_string(), + level: libdd_telemetry::data::LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + )), + ), + ], + }; + sender + .send_actions(direct_log("before first cleanup")) + .await + .expect("queue initial direct action"); + sender.barrier().await.expect("initial receiver barrier"); + + handler_a.cleanup().await; + assert!(server.find_runtime(&instance).is_some()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_some()); + sender + .send_actions(direct_log("after first cleanup")) + .await + .expect("queue direct action for remaining owner"); + sender.barrier().await.expect("remaining-owner barrier"); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_some()); + + handler_b.cleanup().await; + assert!(server.find_runtime(&instance).is_none()); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none()); + receiver_task.abort(); + } + + #[tokio::test] + async fn overlapping_runtime_retirements_detach_after_the_last_finisher() { + let server = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + server.get_runtime(&instance); + let instances = HashSet::from([instance.clone()]); + + let first_retirement = server + .direct_telemetry_lifecycles + .begin_retire_runtimes(&instances); + let second_retirement = server + .direct_telemetry_lifecycles + .begin_retire_runtimes(&instances); + server + .direct_telemetry_lifecycles + .retire_runtimes(&first_retirement); + server + .direct_telemetry_lifecycles + .retire_runtimes(&second_retirement); + + let first: Option = + server + .direct_telemetry_lifecycles + .finish_retire_runtimes(&first_retirement, |ready| { + assert!(ready.is_empty()); + None + }); + assert!(first.is_none()); + assert!(server.find_runtime(&instance).is_some()); + server.get_runtime(&instance); + assert!(server.direct_telemetry_lifecycles.is_retiring(&instance)); + + let second = server.direct_telemetry_lifecycles.finish_retire_runtimes( + &second_retirement, + |ready| { + assert_eq!(ready, &instances); + server + .find_session(&instance.session_id) + .and_then(|session| session.take_runtime(&instance.runtime_id)) + }, + ); + assert!(second.is_some()); + assert!(server.find_runtime(&instance).is_none()); + + server.get_runtime(&instance); + assert!(server.find_runtime(&instance).is_some()); + assert!(!server.direct_telemetry_lifecycles.is_retiring(&instance)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn delayed_runtime_retirement_does_not_retire_reused_session_generation() { + const SERVICE: &str = "cross-scope-retirement"; + const ENV: &str = "test"; + + let server = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + server.get_runtime(&instance); + let instances = HashSet::from([instance.clone()]); + let runtime_retirement = server + .direct_telemetry_lifecycles + .begin_retire_runtimes(&instances); + + server + .direct_telemetry_lifecycles + .begin_retire_session(&instance.session_id); + server + .direct_telemetry_lifecycles + .retire_session(&instance.session_id); + let retired_session = server + .direct_telemetry_lifecycles + .finish_retire_session(&instance.session_id, || { + server.sessions.lock_or_panic().remove(&instance.session_id) + }) + .flatten(); + assert!(retired_session.is_some()); + + let replacement_session = server.get_session(&instance.session_id); + server.get_runtime(&instance); + *replacement_session.session_config.lock_or_panic() = Some(Config::default()); + + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + server.install_direct_telemetry_sender(sender.clone()); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(server.clone(), receiver)); + sender + .retire(DirectTelemetryRetirement::Runtimes( + runtime_retirement.clone(), + )) + .await + .expect("resume delayed runtime retirement"); + server + .direct_telemetry_lifecycles + .finish_retire_runtimes(&runtime_retirement, |ready| { + assert!(ready.is_empty()); + }); + + assert!(!server.direct_telemetry_lifecycles.is_retiring(&instance)); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::AddLog(( + libdd_telemetry::worker::LogIdentifier { identifier: 1 }, + libdd_telemetry::data::Log { + message: "replacement remains active".to_string(), + level: libdd_telemetry::data::LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + )), + ), + ], + }) + .await + .expect("queue direct action for replacement"); + sender + .barrier() + .await + .expect("replacement receiver barrier"); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_some()); + receiver_task.abort(); + } + + #[tokio::test] + async fn overlapping_session_retirements_detach_after_the_last_finisher() { + let server = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + server.get_runtime(&instance); + + server + .direct_telemetry_lifecycles + .begin_retire_session(&instance.session_id); + server + .direct_telemetry_lifecycles + .begin_retire_session(&instance.session_id); + server + .direct_telemetry_lifecycles + .retire_session(&instance.session_id); + server + .direct_telemetry_lifecycles + .retire_session(&instance.session_id); + + let first = server + .direct_telemetry_lifecycles + .finish_retire_session(&instance.session_id, || { + server.sessions.lock_or_panic().remove(&instance.session_id) + }); + assert!(first.is_none()); + assert!(server.find_session(&instance.session_id).is_some()); + server.get_runtime(&instance); + assert!(server.direct_telemetry_lifecycles.is_retiring(&instance)); + + let second = server + .direct_telemetry_lifecycles + .finish_retire_session(&instance.session_id, || { + server.sessions.lock_or_panic().remove(&instance.session_id) + }) + .flatten(); + assert!(second.is_some()); + assert!(server.find_session(&instance.session_id).is_none()); + + server.get_runtime(&instance); + assert!(server.find_runtime(&instance).is_some()); + assert!(!server.direct_telemetry_lifecycles.is_retiring(&instance)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn runtime_cleanup_is_terminal_relative_to_direct_action_receiver() { + const SERVICE: &str = "receiver-runtime-cleanup"; + const ENV: &str = "test"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let queued_log = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("queued before cleanup"); + then.status(202); + }) + .await; + let late_log = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("queued after cleanup"); + then.status(202); + }) + .await; + let hook = crate::service::telemetry::MetricRegistrationSnapshotHook::new(); + let server = SidecarServer { + metrics_logs_clients: MetricsLogsClientSet::with_registration_snapshot_hook( + hook.clone(), + ), + ..Default::default() + }; + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + server.install_direct_telemetry_sender(sender.clone()); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(server.clone(), receiver)); + + let instance = InstanceId::new("session", "runtime"); + let session = server.get_session(&instance.session_id); + server.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some({ + let mut config = Config::default(); + config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + config + }); + + let direct_log = |message: &str| InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::AddLog(( + libdd_telemetry::worker::LogIdentifier { identifier: 1 }, + libdd_telemetry::data::Log { + message: message.to_string(), + level: libdd_telemetry::data::LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + )), + ), + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::Lifecycle(LifecycleAction::FlushData), + ), + ], + }; + sender + .send_actions(direct_log("in flight")) + .await + .expect("queue in-flight direct action"); + hook.wait_until_snapshot(); + sender + .send_actions(direct_log("queued before cleanup")) + .await + .expect("queue direct action behind in-flight work"); + + let cleanup_server = server.clone(); + let cleanup_instance = instance.clone(); + let cleanup = + tokio::spawn(async move { cleanup_server.stop_runtime(&cleanup_instance).await }); + timeout(TokioDuration::from_secs(1), async { + while !server.direct_telemetry_lifecycles.is_retiring(&instance) { + tokio::task::yield_now().await; + } + }) + .await + .expect("runtime retirement should begin"); + hook.resume_creation(); + cleanup.await.expect("runtime cleanup task"); + timeout(TokioDuration::from_secs(5), async { + while queued_log.calls_async().await != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("queued pre-cleanup action should be delivered"); + + sender + .send_actions(direct_log("queued after cleanup")) + .await + .expect("queue late direct action"); + sender.barrier().await.expect("receiver barrier"); + + assert_eq!(late_log.calls_async().await, 0); + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none()); + assert!(session.lock_runtimes().get(&instance.runtime_id).is_none()); + receiver_task.abort(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn session_cleanup_is_terminal_relative_to_direct_action_receiver() { + const SERVICE: &str = "receiver-session-cleanup"; + const ENV: &str = "test"; + const METRIC: &str = "receiver.session.metric"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let queued_log = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("queued before session cleanup"); + then.status(202); + }) + .await; + let hook = crate::service::telemetry::MetricRegistrationSnapshotHook::new(); + let server = SidecarServer { + metrics_logs_clients: MetricsLogsClientSet::with_registration_snapshot_hook( + hook.clone(), + ), + ..Default::default() + }; + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + server.install_direct_telemetry_sender(sender.clone()); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(server.clone(), receiver)); + + let instance = InstanceId::new("session", "runtime"); + let session = server.get_session(&instance.session_id); + server.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some({ + let mut config = Config::default(); + config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + config + }); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::RegisterTelemetryMetric( + MetricContext { + name: METRIC.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + ), + crate::service::telemetry::InternalTelemetryAction::AddMetricPoint(( + 1.0, + METRIC.to_string(), + Vec::new(), + )), + ], + }) + .await + .expect("queue in-flight metric registration"); + hook.wait_until_snapshot(); + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::AddLog(( + libdd_telemetry::worker::LogIdentifier { identifier: 2 }, + libdd_telemetry::data::Log { + message: "queued before session cleanup".to_string(), + level: libdd_telemetry::data::LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + )), + ), + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::Lifecycle(LifecycleAction::FlushData), + ), + ], + }) + .await + .expect("queue direct action behind in-flight session work"); + + let cleanup_server = server.clone(); + let cleanup = tokio::spawn(async move { cleanup_server.stop_session("session").await }); + timeout(TokioDuration::from_secs(1), async { + while !server.direct_telemetry_lifecycles.is_retiring(&instance) { + tokio::task::yield_now().await; + } + }) + .await + .expect("session retirement should begin"); + hook.resume_creation(); + cleanup.await.expect("session cleanup task"); + timeout(TokioDuration::from_secs(5), async { + while queued_log.calls_async().await != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("queued pre-cleanup session action should be delivered"); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::RegisterTelemetryMetric( + MetricContext { + name: "late.metric".to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + ), + ], + }) + .await + .expect("queue late session action"); + sender.barrier().await.expect("receiver barrier"); + + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none()); + assert!(server + .metrics_logs_clients + .registered_metrics(&instance, SERVICE, ENV) + .is_empty()); + assert!(server + .sessions + .lock_or_panic() + .get(&instance.session_id) + .is_none()); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn closed_direct_receiver_falls_back_to_synchronous_cleanup() { + const SERVICE: &str = "closed-receiver-cleanup"; + const ENV: &str = "test"; + + let server = SidecarServer::default(); + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + drop(receiver); + server.install_direct_telemetry_sender(sender); + + let instance = InstanceId::new("session", "runtime"); + let session = server.get_session(&instance.session_id); + session.get_runtime(&instance.runtime_id); + server.metrics_logs_clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance, + &RuntimeMetadata::new("php", "8.3", "test"), + Config::default, + Vec::new(), + ); + + server.stop_runtime(&instance).await; + + assert!(server + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none()); + assert!(session.find_runtime(&instance.runtime_id).is_none()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn non_fork_reconfiguration_recreates_direct_worker_with_new_endpoint() { + const SERVICE: &str = "reconfigured-direct-worker"; + const ENV: &str = "test"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let old_server = MockServer::start_async().await; + let new_server = MockServer::start_async().await; + let old_log = old_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("before-reconfiguration"); + then.status(202); + }) + .await; + let stale_new_log = old_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("after-reconfiguration"); + then.status(202); + }) + .await; + let new_log = new_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("after-reconfiguration"); + then.status(202); + }) + .await; + + let hook = crate::service::telemetry::MetricRegistrationSnapshotHook::new(); + let server = SidecarServer { + metrics_logs_clients: MetricsLogsClientSet::with_registration_snapshot_hook( + hook.clone(), + ), + ..Default::default() + }; + let (sender, receiver) = crate::service::telemetry::direct_telemetry_channel(&server); + server.install_direct_telemetry_sender(sender.clone()); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(server.clone(), receiver)); + let handler = test_handler(server.clone()); + handler + .set_session_config( + "session".to_string(), + #[cfg(windows)] + crate::service::RemoteConfigNotifyFunction::default(), + test_session_config( + Endpoint { + url: old_server.url("/").parse().unwrap(), + ..Endpoint::default() + }, + SERVICE, + ), + true, + ) + .await; + + let instance = InstanceId::new("session", "runtime"); + server.get_runtime(&instance); + let direct_log = |message: &str| InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::AddLog(( + libdd_telemetry::worker::LogIdentifier { identifier: 7 }, + libdd_telemetry::data::Log { + message: message.to_string(), + level: libdd_telemetry::data::LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + )), + ), + crate::service::telemetry::InternalTelemetryAction::TelemetryAction( + TelemetryActions::Lifecycle(LifecycleAction::FlushData), + ), + ], + }; + sender + .send_actions(direct_log("in-flight reconfiguration blocker")) + .await + .unwrap(); + hook.wait_until_snapshot(); + sender + .send_actions(direct_log("before-reconfiguration")) + .await + .unwrap(); + let new_endpoint = new_server.url("/"); + let reconfiguration = tokio::spawn(async move { + handler + .set_session_config( + "session".to_string(), + #[cfg(windows)] + crate::service::RemoteConfigNotifyFunction::default(), + test_session_config( + Endpoint { + url: new_endpoint.parse().unwrap(), + ..Endpoint::default() + }, + SERVICE, + ), + false, + ) + .await; + }); + timeout(TokioDuration::from_secs(5), async { + while !server.direct_telemetry_lifecycles.is_retiring(&instance) { + tokio::task::yield_now().await; + } + }) + .await + .expect("reconfiguration retirement should begin"); + hook.resume_creation(); + reconfiguration.await.expect("reconfiguration task"); + timeout(TokioDuration::from_secs(5), async { + while old_log.calls_async().await != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("queued action should reach the original endpoint"); + + server.get_runtime(&instance); + sender + .send_actions(direct_log("after-reconfiguration")) + .await + .unwrap(); + timeout(TokioDuration::from_secs(5), async { + while new_log.calls_async().await != 1 { + tokio::task::yield_now().await; + } + }) + .await + .expect("replacement worker should deliver to the new endpoint"); + + assert_eq!(old_log.calls_async().await, 1); + assert_eq!(stale_new_log.calls_async().await, 0); + assert_eq!(new_log.calls_async().await, 1); + receiver_task.abort(); + } + + fn ffe_context() -> FfeTelemetryContext { + FfeTelemetryContext { + service: "svc".to_owned(), + env: "prod".to_owned(), + version: "1".to_owned(), + } + } + + fn ffe_exposure(subject_id: &str) -> FfeExposure { + FfeExposure { + timestamp_ms: 123, + flag_key: "flag".to_owned(), + subject_id: subject_id.to_owned(), + subject_attributes_json: "{}".to_owned(), + allocation_key: "alloc".to_owned(), + variant: "variant".to_owned(), + } + } + + fn ffe_metric() -> FfeEvaluationMetric { + FfeEvaluationMetric { + flag_key: "flag".to_owned(), + variant: "variant".to_owned(), + reason: "TARGETING_MATCH".to_owned(), + error_type: None, + allocation_key: Some("alloc".to_owned()), + } + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn ffe_exposure_actions_dispatch_without_registered_application() { + let http_server = MockServer::start_async().await; + let exposures_mock = http_server + .mock_async(|when, then| { + when.method(POST) + .path(ffe_exposures_flusher::EVP_EXPOSURES_PATH); + then.status(202); + }) + .await; let handler = test_handler(SidecarServer::default()); let instance_id = InstanceId::new("session", "runtime"); @@ -1412,73 +2596,692 @@ mod tests { assert_eq!(metrics_mock.calls_async().await, 0); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[tokio::test] #[cfg_attr(miri, ignore)] - async fn buffered_initial_data_reaches_app_started_through_enqueue_actions() { - const CLIENTS: usize = 16; + async fn composer_before_config_waits_for_configured_app_started() { + const SERVICE: &str = "composer-before-config"; + const ENV: &str = "test"; const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; let http_server = MockServer::start_async().await; - let app_started_with_initial_data = http_server + let configured_start = http_server .mock_async(|when, then| { when.method(POST) .path(TELEMETRY_PATH) .body_includes("\"request_type\":\"app-started\"") - .body_includes("\"name\":\"race_config\"") - .body_includes("\"name\":\"race_config_second\"") - .body_includes("\"name\":\"startup-dependency\"") - .body_includes("\"name\":\"startup-integration\""); + .body_includes("\"name\":\"php_config\""); then.status(202); }) .await; - let app_started_without_initial_data = http_server + let empty_start = http_server .mock_async(|when, then| { when.method(POST) .path(TELEMETRY_PATH) - .body_includes("\"request_type\":\"app-started\""); + .body_includes("\"request_type\":\"app-started\"") + .body_excludes("\"name\":\"php_config\""); then.status(202); }) .await; let handler = test_handler(SidecarServer::default()); - let session = handler.server.get_session("session"); - let mut telemetry_config = Config::default(); - telemetry_config - .set_endpoint_uri(http_server.url("/").parse().unwrap()) - .unwrap(); - *session.session_config.lock_or_panic() = Some(telemetry_config.clone()); + let instance_id = InstanceId::new("session", "runtime"); + let queue_id = QueueId::from(1); + let session = handler.server.get_session(&instance_id.session_id); + *session.session_config.lock_or_panic() = Some({ + let mut config = Config::default(); + config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + config + }); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + ENV.to_string(), + String::new(), + SERVICE.to_string(), + Vec::new(), + ); - for index in 0..CLIENTS { - let service = format!("telemetry-enqueue-race-{index}"); - let instance_id = InstanceId::new("session", &format!("runtime-{index}")); - let queue_id = QueueId::from(index as u64 + 1); - handler - .server - .get_runtime(&instance_id) - .lock_applications() - .entry(queue_id) - .or_default() - .set_metadata(String::new(), String::new(), service, Vec::new()); - handler.server.metrics_logs_clients.get_or_create( - &format!("telemetry-enqueue-race-{index}"), - "", - &instance_id, + handler + .enqueue_actions( + instance_id.clone(), + queue_id, + vec![SidecarAction::PhpComposerTelemetryFile(PathBuf::from( + "/missing/vendor/composer/installed.json", + ))], + ) + .await; + sleep(TokioDuration::from_millis(50)).await; + assert_eq!(configured_start.calls_async().await, 0); + assert_eq!(empty_start.calls_async().await, 0); + + handler + .enqueue_actions( + instance_id, + queue_id, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "php_config".to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + ))], + ) + .await; + + timeout(TokioDuration::from_secs(5), async { + while configured_start.calls_async().await != 1 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("configured app-started request"); + assert_eq!(empty_start.calls_async().await, 0); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn composer_before_terminal_stop_is_delivered_before_app_closing() { + const SERVICE: &str = "composer-before-terminal-stop"; + const ENV: &str = "test"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let app_started = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\""); + then.status(202); + }) + .await; + let terminal_batch = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-dependencies-loaded\"") + .body_includes("\"name\":\"datadog/dd-trace\"") + .body_includes("\"request_type\":\"app-closing\""); + then.status(202); + }) + .await; + + let handler = test_handler(SidecarServer::default()); + let instance_id = InstanceId::new("session", "runtime"); + let queue_id = QueueId::from(1); + let session = handler.server.get_session(&instance_id.session_id); + *session.session_config.lock_or_panic() = Some({ + let mut config = Config::default(); + config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + config + }); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + ENV.to_string(), + String::new(), + SERVICE.to_string(), + Vec::new(), + ); + + handler + .enqueue_actions( + instance_id.clone(), + queue_id, + vec![SidecarAction::PhpComposerTelemetryFile( + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fixtures/installed.json"), + )], + ) + .await; + handler + .enqueue_actions( + instance_id, + queue_id, + vec![SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop, + ))], + ) + .await; + + let delivered = timeout(TokioDuration::from_secs(5), async { + loop { + if app_started.calls_async().await == 1 && terminal_batch.calls_async().await == 1 { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + if delivered.is_err() { + panic!( + "expected Start=1 and Composer+Stop batch=1; observed Start={}, batch={}", + app_started.calls_async().await, + terminal_batch.calls_async().await, + ) + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg_attr(miri, ignore)] + async fn stop_race_returns_untouched_actions_to_the_next_lifecycle() { + const SERVICE: &str = "stop-enqueue-race"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let instance = InstanceId::new("session", "runtime"); + let active = clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &instance, + &runtime_metadata, + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + ) + .expect("active application telemetry worker"); + let selected = Arc::new(Barrier::new(2)); + let resume = Arc::new(Barrier::new(2)); + + let racing_clients = clients.clone(); + let racing_runtime = runtime_metadata.clone(); + let racing_instance = instance.clone(); + let selected_by_enqueue = selected.clone(); + let resume_enqueue = resume.clone(); + let enqueue = tokio::spawn(async move { + let dispatch = racing_clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &racing_instance, + &racing_runtime, + Vec::new(), + ), + PendingApplicationAction::from_actions( + &racing_instance, + vec![SidecarAction::Telemetry(TelemetryActions::AddDependency( + libdd_telemetry::data::Dependency { + name: "survives-stop-race".to_string(), + version: None, + }, + ))], + &HashMap::new(), + ), + Config::default, + |_, _| panic!("the active lifecycle should already exist"), + ); + let ApplicationTelemetryDispatch::Ready { + client, + actions, + created, + .. + } = dispatch + else { + panic!("the racing batch should select the active lifecycle"); + }; + selected_by_enqueue.wait().await; + resume_enqueue.wait().await; + schedule_application_actions( + &client, + actions, + created, + &racing_instance, + QueueId::from(1), + ) + .expect_err("a stopped lifecycle must return the untouched owned actions") + }); + + selected.wait().await; + let stop_result = schedule_application_actions( + &active, + PendingApplicationAction::from_actions( + &instance, + vec![SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop, + ))], + &HashMap::new(), + ), + false, + &instance, + QueueId::from(1), + ) + .expect("Stop should schedule on the selected active lifecycle"); + assert!(stop_result.remove_client); + clients.remove_telemetry_client(SERVICE, ENV, &active); + resume.wait().await; + + let returned_actions = enqueue.await.expect("racing enqueue task"); + assert_eq!(returned_actions.len(), 1); + let promoted = Arc::new(Mutex::new(Vec::new())); + let promoted_by_initializer = promoted.clone(); + let mut next_actions = returned_actions; + next_actions.extend(PendingApplicationAction::from_actions( + &instance, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "next-lifecycle".to_string(), + value: "configured".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + ))], + &HashMap::new(), + )); + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime_metadata, Vec::new()), + next_actions, + Config::default, + move |_, actions| { + promoted_by_initializer + .lock_or_panic() + .extend(actions.into_iter().map(|pending| pending.action)); + false + }, + ); + assert!(matches!( + dispatch, + ApplicationTelemetryDispatch::Ready { created: true, .. } + )); + assert!(promoted.lock_or_panic().iter().any(|action| matches!( + action, + SidecarAction::Telemetry(TelemetryActions::AddDependency(dependency)) + if dependency.name == "survives-stop-race" + ))); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn terminal_initialization_returns_suffix_to_the_next_lifecycle() { + const SERVICE: &str = "initial-stop-suffix"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let instance = InstanceId::new("session", "runtime"); + let next_configuration = libdd_telemetry::data::Configuration { + name: "next-lifecycle".to_string(), + value: "configured".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }; + let actions = PendingApplicationAction::from_actions( + &instance, + vec![ + SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)), + SidecarAction::Telemetry(TelemetryActions::AddConfig(next_configuration)), + ], + &HashMap::new(), + ); + + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime_metadata, Vec::new()), + actions, + Config::default, + |client, actions| { + schedule_application_actions(client, actions, true, &instance, QueueId::from(1)) + .expect("the initial lifecycle should accept its terminal prefix") + .remove_client + }, + ); + + let ApplicationTelemetryDispatch::Ready { + actions, + remove_client, + .. + } = dispatch + else { + panic!("Stop should promote the initial lifecycle"); + }; + assert!(remove_client); + assert_eq!(actions.len(), 1); + assert!(matches!( + &actions[0].action, + SidecarAction::Telemetry(TelemetryActions::AddConfig(configuration)) + if configuration.name == "next-lifecycle" + )); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn non_state_application_action_retries_due_shared_memory_creation() { + const SERVICE: &str = "retry-on-dependency"; + const ENV: &str = "test"; + + let retry_started = Instant::now() - Duration::from_secs(60); + let client = TelemetryCachedClient::new_with_shm_factory( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "runtime"), &RuntimeMetadata::new("php", "8.3", "test"), - || telemetry_config.clone(), + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + retry_started, + |_| Err(std::io::Error::other("injected initial failure")), + ) + .expect("worker should start even when initial SHM creation fails"); + let client = Arc::new(Mutex::new(Some(client))); + + let _ = schedule_application_actions( + &client, + PendingApplicationAction::from_actions( + &InstanceId::new("session", "runtime"), + vec![SidecarAction::Telemetry(TelemetryActions::AddDependency( + libdd_telemetry::data::Dependency { + name: "retry-trigger".to_string(), + version: None, + }, + ))], + &HashMap::new(), + ), + false, + &InstanceId::new("session", "runtime"), + QueueId::from(1), + ); + + assert!(matches!( + client.lock_or_panic().as_ref(), + Some(application_client) if application_client.has_ready_shm() + )); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn pending_startup_data_is_seeded_before_start() { + const SERVICE: &str = "startup-data-before-config"; + const ENV: &str = "test"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let any_start = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\""); + then.status(202); + }) + .await; + let complete_start = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_includes("\"name\":\"startup-dependency\"") + .body_includes("\"name\":\"startup-integration\"") + .body_includes("\"name\":\"startup-config\""); + then.status(202); + }) + .await; + let dependency_change = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-dependencies-loaded\""); + then.status(202); + }) + .await; + let integration_change = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-integrations-change\""); + then.status(202); + }) + .await; + + let handler = test_handler(SidecarServer::default()); + let instance_id = InstanceId::new("session", "runtime"); + let queue_id = QueueId::from(1); + let session = handler.server.get_session(&instance_id.session_id); + *session.session_config.lock_or_panic() = Some({ + let mut config = Config::default(); + config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + config + }); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + ENV.to_string(), + String::new(), + SERVICE.to_string(), Vec::new(), ); - let configuration = |name: &str| { - SidecarAction::Telemetry(TelemetryActions::AddConfig( + handler + .enqueue_actions( + instance_id.clone(), + queue_id, + vec![ + SidecarAction::Telemetry(TelemetryActions::AddDependency( + libdd_telemetry::data::Dependency { + name: "startup-dependency".to_string(), + version: None, + }, + )), + SidecarAction::Telemetry(TelemetryActions::AddIntegration( + libdd_telemetry::data::Integration { + name: "startup-integration".to_string(), + enabled: true, + version: None, + compatible: None, + auto_enabled: None, + }, + )), + ], + ) + .await; + sleep(TokioDuration::from_millis(50)).await; + assert_eq!(any_start.calls_async().await, 0); + assert_eq!(complete_start.calls_async().await, 0); + assert_eq!(dependency_change.calls_async().await, 0); + assert_eq!(integration_change.calls_async().await, 0); + any_start.delete_async().await; + + handler + .enqueue_actions( + instance_id, + queue_id, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( libdd_telemetry::data::Configuration { - name: name.to_string(), + name: "startup-config".to_string(), value: "present".to_string(), origin: libdd_telemetry::data::ConfigurationOrigin::Default, config_id: None, seq_id: None, }, - )) - }; + ))], + ) + .await; + + timeout(TokioDuration::from_secs(5), async { + while complete_start.calls_async().await != 1 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("complete app-started request"); + + let worker = handler + .server + .telemetry_clients + .workers() + .into_iter() + .next() + .expect("application telemetry worker"); + worker + .send_msg(TelemetryActions::Lifecycle(LifecycleAction::FlushData)) + .await + .unwrap(); + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .unwrap(); + rx.await.unwrap(); + + assert_eq!(complete_start.calls_async().await, 1); + assert_eq!(dependency_change.calls_async().await, 0); + assert_eq!(integration_change.calls_async().await, 0); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn pending_metric_point_uses_originating_runtime_registration() { + const SERVICE: &str = "pending-metric-origin"; + const ENV: &str = "test"; + const METRIC: &str = "originating.runtime.metric"; + + let server = SidecarServer::default(); + let origin_handler = test_handler(server.clone()); + let promoting_handler = test_handler(server.clone()); + let origin_instance = InstanceId::new("session", "origin-runtime"); + let promoting_instance = InstanceId::new("session", "promoting-runtime"); + let origin_queue = QueueId::from(1); + let promoting_queue = QueueId::from(2); + + *server.get_session("session").session_config.lock_or_panic() = Some(Config::default()); + + for (instance_id, queue_id) in [ + (&origin_instance, origin_queue), + (&promoting_instance, promoting_queue), + ] { + server + .get_runtime(instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + ENV.to_string(), + String::new(), + SERVICE.to_string(), + Vec::new(), + ); + } + + origin_handler + .register_telemetry_metric(MetricContext { + name: METRIC.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }) + .await; + origin_handler + .enqueue_actions( + origin_instance, + origin_queue, + vec![SidecarAction::AddTelemetryMetricPoint(( + METRIC.to_string(), + 1.0, + Vec::new(), + ))], + ) + .await; + + promoting_handler + .enqueue_actions( + promoting_instance, + promoting_queue, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "startup-config".to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + ))], + ) + .await; + + let app_client = server + .telemetry_clients + .clients() + .into_iter() + .next() + .expect("promoting config should create the application worker"); + assert!(app_client + .lock_or_panic() + .as_ref() + .expect("application telemetry client") + .telemetry_metrics + .contains_key(METRIC)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn initial_config_reaches_app_started_through_enqueue_actions() { + const CLIENTS: usize = 16; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let app_started_with_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_includes("\"name\":\"race_config\"") + .body_includes("\"name\":\"race_config_second\""); + then.status(202); + }) + .await; + let app_started_without_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_excludes("\"name\":\"race_config\""); + then.status(202); + }) + .await; + + let handler = test_handler(SidecarServer::default()); + let session = handler.server.get_session("session"); + let mut telemetry_config = Config::default(); + telemetry_config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + *session.session_config.lock_or_panic() = Some(telemetry_config); + + for index in 0..CLIENTS { + let service = format!("telemetry-enqueue-race-{index}"); + let instance_id = InstanceId::new("session", &format!("runtime-{index}")); + let queue_id = QueueId::from(index as u64 + 1); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata(String::new(), String::new(), service, Vec::new()); + handler .enqueue_actions( instance_id, @@ -1490,99 +3293,343 @@ mod tests { version: None, }, )), - SidecarAction::Telemetry(TelemetryActions::AddIntegration( - libdd_telemetry::data::Integration { - name: "startup-integration".to_string(), - enabled: true, - ..Default::default() + SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "race_config".to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + )), + SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "race_config_second".to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, }, )), - configuration("race_config"), - configuration("race_config_second"), ], ) .await; } tokio::time::timeout(TokioDuration::from_secs(10), async { - while app_started_with_initial_data.calls_async().await - + app_started_without_initial_data.calls_async().await - != CLIENTS - { + loop { + let with_config = app_started_with_config.calls_async().await; + let without_config = app_started_without_config.calls_async().await; + if with_config + without_config == CLIENTS { + break; + } + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("all app-started requests should arrive"); + + let missing = app_started_without_config.calls_async().await; + assert_eq!( + missing, 0, + "{missing} app-started payloads raced ahead of their initial config" + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn stats_concentrator_does_not_start_app_telemetry_before_config() { + const SERVICE: &str = "stats-before-config"; + const ENV: &str = "test-env"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let app_started_with_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_includes("\"name\":\"stats_race_config\""); + then.status(202); + }) + .await; + let app_started_without_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_excludes("\"name\":\"stats_race_config\""); + then.status(202); + }) + .await; + + let handler = test_handler(SidecarServer::default()); + let session = handler.server.get_session("session"); + let mut telemetry_config = Config::default(); + telemetry_config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + *session.session_config.lock_or_panic() = Some(telemetry_config); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint::default(), + flush_interval: Duration::from_secs(60), + hostname: String::new(), + process_tags: String::new(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), + }); + + let instance_id = InstanceId::new("session", "runtime"); + let queue_id = QueueId::from(1); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + ENV.to_string(), + String::new(), + SERVICE.to_string(), + Vec::new(), + ); + + let concentrator = + get_or_create_concentrator(&handler.server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + + let worker = concentrator + .telemetry + .as_ref() + .expect("stats telemetry worker"); + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .unwrap(); + rx.await.unwrap(); + assert_eq!( + app_started_without_config.calls_async().await, + 0, + "stats creation started app telemetry before tracer config arrived" + ); + + handler + .enqueue_actions( + instance_id, + queue_id, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: "stats_race_config".to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + ))], + ) + .await; + + tokio::time::timeout(TokioDuration::from_secs(10), async { + while app_started_with_config.calls_async().await != 1 { sleep(TokioDuration::from_millis(10)).await; } }) .await - .expect("all app-started requests should arrive"); + .expect("configured app-started request should arrive"); - assert_eq!( - app_started_with_initial_data.calls_async().await, - CLIENTS, - "every app-started payload should contain the complete initial data batch" - ); - assert_eq!(app_started_without_initial_data.calls_async().await, 0); + assert_eq!(app_started_without_config.calls_async().await, 0); } - #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + #[tokio::test] #[cfg_attr(miri, ignore)] - async fn concurrent_same_key_creation_starts_one_worker() { - const CALLERS: usize = 32; - const SERVICE: &str = "concurrent-client-creation"; - const ENV: &str = "test"; + async fn telemetry_flush_includes_stats_workers() { + const SERVICE: &str = "stats-worker-flush"; + const ENV: &str = "test-env"; + const METRIC: &str = "stats_worker.flush_test"; const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; let http_server = MockServer::start_async().await; - let app_started = http_server + let metric_request = http_server .mock_async(|when, then| { when.method(POST) .path(TELEMETRY_PATH) - .body_includes("\"request_type\":\"app-started\""); + .body_includes(format!("\"metric\":\"{METRIC}\"")); then.status(202); }) .await; - let mut config = Config::default(); - config + + let handler = test_handler(SidecarServer::default()); + let session = handler.server.get_session("session"); + let mut telemetry_config = Config::default(); + telemetry_config .set_endpoint_uri(http_server.url("/").parse().unwrap()) .unwrap(); + *session.session_config.lock_or_panic() = Some(telemetry_config); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint::default(), + flush_interval: Duration::from_secs(60), + hostname: String::new(), + process_tags: String::new(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), + }); - let clients = TelemetryCachedClientSet::default(); - let barrier = Arc::new(Barrier::new(CALLERS)); - let tasks = (0..CALLERS).map(|index| { - let clients = clients.clone(); - let barrier = barrier.clone(); - let config = config.clone(); - tokio::spawn(async move { - let instance_id = InstanceId::new("session", &format!("runtime-{index}")); - barrier.wait().await; - clients.get_or_create( - SERVICE, - ENV, - &instance_id, - &RuntimeMetadata::new("php", "8.3", "test"), - || config, - Vec::new(), - ) + let state = + get_or_create_concentrator(&handler.server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + let worker = state.telemetry.as_ref().expect("stats telemetry worker"); + let context = worker.register_metric_context( + METRIC.to_string(), + Vec::new(), + libdd_telemetry::data::metrics::MetricType::Count, + true, + libdd_telemetry::data::metrics::MetricNamespace::Tracers, + ); + worker.add_point(1.0, &context, Vec::new()).unwrap(); + + handler + .flush(SidecarFlushOptions { + traces_and_stats: true, + telemetry: true, }) + .await; + + assert_eq!( + metric_request.calls_async().await, + 1, + "flush returned before the dedicated stats telemetry worker sent its metric" + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn explicit_stats_flush_reuses_exporter_metric_context() { + const SERVICE: &str = "stats-exporter-reuse"; + const ENV: &str = "stats-exporter-reuse-env"; + + let server = SidecarServer::default(); + let session = server.get_session("stats-exporter-reuse-session"); + *session.session_config.lock_or_panic() = Some(Config::default()); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint::default(), + flush_interval: Duration::from_secs(3600), + hostname: String::new(), + process_tags: String::new(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), }); - let returned_clients = futures::future::join_all(tasks) + let state = get_or_create_concentrator(&server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + tokio::task::yield_now().await; + + let worker = state.telemetry.as_ref().expect("stats telemetry worker"); + let before = worker + .stats() + .expect("stats collection request") .await - .into_iter() - .map(Result::unwrap) - .collect::>(); + .expect("stats collection response") + .metric_contexts; - let first = &returned_clients[0]; - assert!( - returned_clients - .iter() - .all(|client| Arc::ptr_eq(first, client)), - "all same-key callers should receive the same telemetry client" + flush_all_stats_now(std::slice::from_ref(&state)).await; + flush_all_stats_now(std::slice::from_ref(&state)).await; + + let after = worker + .stats() + .expect("stats collection request") + .await + .expect("stats collection response") + .metric_contexts; + assert_eq!( + after, before, + "manual flushes must reuse the state-owned exporter telemetry context" ); - let worker = first - .lock_or_panic() - .as_ref() - .expect("telemetry client") - .worker - .clone(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn stats_worker_uses_auxiliary_identity_without_registering_runtime() { + const SERVICE: &str = "stats-auxiliary"; + const ENV: &str = "stats-auxiliary-env"; + const METRIC: &str = "stats_auxiliary.metric"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let server = SidecarServer::default(); + let session = server.get_session("stats-auxiliary-session"); + session.get_runtime(&"application-runtime".to_string()); + *session.process_tags.lock_or_panic() = + vec![Tag::new("custom", "value").expect("valid test tag")]; + *session.auto_resolved_service_name.lock_or_panic() = Some(SERVICE.to_string()); + let expected_process_tags = session + .process_tags_with_svc_source() + .iter() + .map(ToString::to_string) + .collect::>() + .join(","); + let mut telemetry_config = Config::default(); + telemetry_config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + *session.session_config.lock_or_panic() = Some(telemetry_config); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint::default(), + flush_interval: Duration::from_secs(3600), + hostname: String::new(), + process_tags: expected_process_tags.clone(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), + }); + let runtime_count_before = session.lock_runtimes().len(); + let sidecar_runtime_count_before = server.compute_stats().await.runtimes; + + let state = get_or_create_concentrator(&server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + + assert_eq!(session.lock_runtimes().len(), runtime_count_before); + assert_eq!( + server.compute_stats().await.runtimes, + sidecar_runtime_count_before, + "the auxiliary stats worker must not appear in SidecarStats.runtimes" + ); + assert!(state.meta.runtime_id.starts_with("stats-")); + assert_ne!(state.meta.runtime_id, "application-runtime"); + assert_ne!(state.meta.runtime_id, "caller-runtime"); + assert_eq!(state.meta.process_tags, expected_process_tags); + + let metric_request = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes(format!("\"metric\":\"{METRIC}\"")) + .body_includes(format!("\"runtime_id\":\"{}\"", state.meta.runtime_id)) + .body_includes(format!("\"process_tags\":\"{expected_process_tags}\"")); + then.status(202); + }) + .await; + let worker = state.telemetry.as_ref().expect("stats telemetry worker"); + let context = worker.register_metric_context( + METRIC.to_string(), + Vec::new(), + libdd_telemetry::data::metrics::MetricType::Count, + true, + libdd_telemetry::data::metrics::MetricNamespace::Tracers, + ); + worker.add_point(1.0, &context, Vec::new()).unwrap(); + worker + .send_msg(TelemetryActions::Lifecycle( + LifecycleAction::FlushMetricAggr, + )) + .await + .unwrap(); + worker + .send_msg(TelemetryActions::Lifecycle(LifecycleAction::FlushData)) + .await + .unwrap(); let (tx, rx) = futures::channel::oneshot::channel(); worker .send_msg(TelemetryActions::CollectStats(tx)) @@ -1590,8 +3637,222 @@ mod tests { .unwrap(); rx.await.unwrap(); - assert_eq!(clients.inner.lock_or_panic().len(), 1); - assert_eq!(app_started.calls_async().await, 1); + assert_eq!( + metric_request.calls_async().await, + 1, + "the auxiliary worker should use the synthetic runtime id and full process tags" + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn stats_flush_snapshots_concentrators_after_trace_flush() { + const SERVICE: &str = "concurrent-stats-worker"; + const ENV: &str = "test-env"; + + let trace_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let trace_address = trace_listener.local_addr().unwrap(); + let (trace_started_tx, trace_started_rx) = oneshot::channel(); + let (release_trace_tx, release_trace_rx) = oneshot::channel(); + let trace_server = tokio::spawn(async move { + let (mut stream, _) = trace_listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(stream.read(&mut request).await.unwrap() > 0); + trace_started_tx.send(()).unwrap(); + release_trace_rx.await.unwrap(); + stream + .write_all( + b"HTTP/1.1 202 Accepted\r\n\ + Content-Type: application/json\r\n\ + Content-Length: 15\r\n\ + Connection: close\r\n\ + \r\n\ + {\"status\":\"ok\"}", + ) + .await + .unwrap(); + }); + let stats_server = MockServer::start_async().await; + let stats_request = stats_server + .mock_async(|when, then| { + when.method(POST).path("/v0.6/stats"); + then.status(202); + }) + .await; + + let server = SidecarServer::default(); + let trace_endpoint = Endpoint { + url: format!("http://{trace_address}/").parse().unwrap(), + ..Endpoint::default() + }; + server + .trace_flusher + .enqueue(create_send_data(128, &trace_endpoint)); + let handler = test_handler(server.clone()); + let flush = tokio::spawn(async move { + handler + .flush(SidecarFlushOptions { + traces_and_stats: true, + telemetry: false, + }) + .await; + }); + + tokio::time::timeout(TokioDuration::from_secs(5), trace_started_rx) + .await + .expect("trace flush should be in flight") + .expect("trace server should report the request"); + + let session = server.get_session("session"); + *session.session_config.lock_or_panic() = Some(Config::default()); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint { + url: stats_server.url("/v0.6/stats").parse().unwrap(), + ..Endpoint::default() + }, + flush_interval: Duration::from_secs(60), + hostname: String::new(), + process_tags: String::new(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), + }); + let state = get_or_create_concentrator(&server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + state.concentrator.add_span(&ShmSpanInput { + fixed: FixedAggregationKey { + service_name: SERVICE, + resource_name: "resource", + operation_name: "operation", + span_type: "web", + span_kind: "server", + http_method: "GET", + http_endpoint: "/", + service_source: "", + http_status_code: 200, + is_synthetics_request: false, + is_trace_root: Default::default(), + grpc_status_code: None, + }, + peer_tags: &[], + duration_ns: 1_000_000, + is_error: false, + is_top_level: true, + }); + + release_trace_tx + .send(()) + .expect("trace response should still be blocked"); + flush.await.unwrap(); + trace_server.await.unwrap(); + assert_eq!( + stats_request.calls_async().await, + 1, + "a concentrator created during trace flush should be included" + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn compute_stats_preserves_application_client_count() { + const SERVICE: &str = "worker-stats"; + const ENV: &str = "test-env"; + + let http_server = MockServer::start_async().await; + let _telemetry = http_server + .mock_async(|when, then| { + when.method(POST) + .path("/telemetry/proxy/api/v2/apmtelemetry"); + then.status(202); + }) + .await; + let server = SidecarServer::default(); + let instance_id = InstanceId::new("session", "runtime"); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let mut telemetry_config = Config::default(); + telemetry_config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + + let app_client = server + .telemetry_clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &instance_id, + &runtime_metadata, + Vec::new(), + ), + || telemetry_config.clone(), + InitialTelemetryData::default(), + ) + .expect("application telemetry worker"); + app_client + .lock_or_panic() + .as_mut() + .expect("application telemetry client") + .register_metric(MetricContext { + name: "app.metric".to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }); + + let metrics_logs_client = server.metrics_logs_clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance_id, + &runtime_metadata, + || telemetry_config.clone(), + Vec::new(), + ); + metrics_logs_client + .lock_or_panic() + .as_mut() + .expect("metrics/logs telemetry client") + .register_metric(MetricContext { + name: "internal.metric".to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }); + + let session = server.get_session(&instance_id.session_id); + *session.session_config.lock_or_panic() = Some(telemetry_config); + *session.stats_config.lock_or_panic() = Some(StatsConfig { + endpoint: Endpoint::default(), + flush_interval: Duration::from_secs(60), + hostname: String::new(), + process_tags: String::new(), + root_service: SERVICE.to_string(), + language: "php".to_string(), + tracer_version: "test".to_string(), + }); + let stats_state = get_or_create_concentrator(&server.span_concentrators, ENV, "", &session) + .expect("stats concentrator"); + stats_state + .telemetry + .as_ref() + .expect("stats telemetry worker") + .register_metric_context( + "stats.metric".to_string(), + Vec::new(), + libdd_telemetry::data::metrics::MetricType::Count, + true, + libdd_telemetry::data::metrics::MetricNamespace::Tracers, + ); + + let stats = server.compute_stats().await; + assert_eq!(stats.active_telemetry_clients, 1); + assert_eq!(stats.telemetry_metrics_contexts, 2); + assert_eq!( + stats.telemetry_worker.metric_contexts, 4, + "application, direct, stats-test, and state-owned exporter contexts are counted" + ); + assert_eq!(stats.telemetry_worker_errors, 0); } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] @@ -1657,6 +3918,7 @@ mod tests { }) .await .expect("app-started request should arrive"); + assert_eq!( app_closing.calls_async().await, 0, @@ -1671,6 +3933,151 @@ mod tests { .await .expect("app-closing request should arrive after app-started"); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn successor_lifecycle_waits_for_predecessor_app_closing() { + const SERVICE: &str = "ordered-successor"; + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + let http_server = MockServer::start_async().await; + let app_started = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\""); + then.status(202); + }) + .await; + let app_closing = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-closing\""); + then.status(202).delay(TokioDuration::from_millis(300)); + }) + .await; + + let handler = Arc::new(test_handler(SidecarServer::default())); + let instance_id = InstanceId::new("session", "successor-runtime"); + let queue_id = QueueId::from(1); + let session = handler.server.get_session(&instance_id.session_id); + let mut telemetry_config = Config::default(); + telemetry_config + .set_endpoint_uri(http_server.url("/").parse().unwrap()) + .unwrap(); + *session.session_config.lock_or_panic() = Some(telemetry_config); + handler + .server + .get_runtime(&instance_id) + .lock_applications() + .entry(queue_id) + .or_default() + .set_metadata( + String::new(), + String::new(), + SERVICE.to_string(), + Vec::new(), + ); + + let configuration = |name: &str| { + SidecarAction::Telemetry(TelemetryActions::AddConfig( + libdd_telemetry::data::Configuration { + name: name.to_string(), + value: "present".to_string(), + origin: libdd_telemetry::data::ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + }, + )) + }; + handler + .enqueue_actions( + instance_id.clone(), + queue_id, + vec![configuration("first-lifecycle")], + ) + .await; + timeout(TokioDuration::from_secs(5), async { + while app_started.calls_async().await != 1 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("first app-started request"); + + let handoff_started = Instant::now(); + handler + .enqueue_actions( + instance_id.clone(), + queue_id, + vec![ + SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)), + configuration("second-lifecycle"), + ], + ) + .await; + assert!( + handoff_started.elapsed() >= TokioDuration::from_millis(300), + "successor dispatch returned before the predecessor app-closing request completed" + ); + assert_eq!(app_closing.calls_async().await, 1); + timeout(TokioDuration::from_secs(5), async { + while app_started.calls_async().await != 2 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("successor app-started request"); + + let stop_handler = handler.clone(); + let stop_instance = instance_id.clone(); + let stop = tokio::spawn(async move { + stop_handler + .enqueue_actions( + stop_instance, + queue_id, + vec![SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop, + ))], + ) + .await; + }); + timeout(TokioDuration::from_secs(5), async { + while app_closing.calls_async().await != 2 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("second predecessor app-closing request"); + + let successor_handler = handler.clone(); + let successor = tokio::spawn(async move { + successor_handler + .enqueue_actions( + instance_id, + queue_id, + vec![configuration("concurrent-successor")], + ) + .await; + }); + sleep(TokioDuration::from_millis(50)).await; + assert_eq!( + app_started.calls_async().await, + 2, + "a concurrent successor bypassed the in-progress terminal handoff" + ); + + stop.await.expect("concurrent stop task"); + successor.await.expect("concurrent successor task"); + timeout(TokioDuration::from_secs(5), async { + while app_started.calls_async().await != 3 { + sleep(TokioDuration::from_millis(10)).await; + } + }) + .await + .expect("concurrent successor app-started request"); + } } // TODO: APMSP-1079 - Unit tests are sparse for the sidecar server. We should add more. diff --git a/datadog-sidecar/src/service/stats_flusher.rs b/datadog-sidecar/src/service/stats_flusher.rs index 030122f09f..eac9fcdb4f 100644 --- a/datadog-sidecar/src/service/stats_flusher.rs +++ b/datadog-sidecar/src/service/stats_flusher.rs @@ -9,7 +9,7 @@ //! automatically once idle: an empty drain sets the `please_reload` bit (telling PHP workers //! to stop writing), and the subsequent flush performs a final drain before removal. -use crate::service::RuntimeMetadata; +use crate::service::{InstanceId, RuntimeMetadata}; use base64::prelude::BASE64_URL_SAFE_NO_PAD; use base64::Engine; use datadog_ipc::shm_stats::{ @@ -79,10 +79,13 @@ pub struct SpanConcentratorState { pub concentrator: ShmSpanConcentrator, /// The stats endpoint (with `/v0.6/stats` path baked in) used by the flush loop. pub(crate) endpoint: Endpoint, - /// Metadata for StatsExporter payload annotation (hostname, env, version, service, …). + /// Metadata retained for identity and payload assertions in regression tests. + #[cfg(test)] pub(crate) meta: StatsMetadata, /// Telemetry client to pass to the stats exporter. pub telemetry: Option, + /// One exporter owns sequence state and its telemetry metric context for this concentrator. + exporter: Arc>, } // SAFETY: ShmSpanConcentrator is designed for cross-process sharing; all internal state @@ -108,19 +111,21 @@ pub fn env_stats_shm_path(env: &str, version: &str, service: &str) -> CString { } fn make_exporter( - s: &SpanConcentratorState, - endpoint: Endpoint, + concentrator: &ShmSpanConcentrator, + meta: &StatsMetadata, + endpoint: &Endpoint, flush_interval: Duration, + telemetry: Option, ) -> StatsExporter { StatsExporter::new( flush_interval, - Arc::new(Mutex::new(s.concentrator.clone())), - s.meta.clone(), - endpoint, + Arc::new(Mutex::new(concentrator.clone())), + meta.clone(), + endpoint.clone(), NativeCapabilities::new_client(), #[cfg(feature = "stats-obfuscation")] "0", - s.telemetry.clone(), + telemetry, None, ) } @@ -148,8 +153,6 @@ pub async fn run_stats_flush_loop( return; }; - let exporter = make_exporter(&state, state.endpoint.clone(), flush_interval); - loop { tokio::time::sleep(flush_interval).await; let Some(arc) = states.upgrade() else { @@ -169,7 +172,7 @@ pub async fn run_stats_flush_loop( ); } - match exporter.send(false).await { + match state.exporter.send(false).await { Err(e) => warn!( "Failed to send stats for env={} version={}: {e}", map_key.env, map_key.version @@ -193,10 +196,7 @@ pub async fn run_stats_flush_loop( guard.remove(&map_key); } } - if let Err(e) = make_exporter(&state, state.endpoint.clone(), flush_interval) - .send(true) - .await - { + if let Err(e) = state.exporter.send(true).await { warn!("Failed final stats flush: {e}"); } break; @@ -215,10 +215,8 @@ pub async fn run_stats_flush_loop( /// Returns `None` when stats config is not available (agentless or not yet configured). pub(crate) fn get_or_create_concentrator( concentrators: &Arc>>>, - telemetry_clients: &crate::service::telemetry::MetricsLogsClientSet, env: &str, version: &str, - runtime_id: &str, session: &crate::service::session_info::SessionInfo, ) -> Option> { let config = session @@ -249,11 +247,12 @@ pub(crate) fn get_or_create_concentrator( let path = env_stats_shm_path(env, version, &service_name); + let auxiliary_runtime_id = format!("stats-{:032x}", rand::random::()); let meta = StatsMetadata { hostname: config.hostname.clone(), env: env.to_owned(), app_version: version.to_owned(), - runtime_id: runtime_id.to_owned(), + runtime_id: auxiliary_runtime_id.clone(), language: config.language.clone(), tracer_version: config.tracer_version.clone(), process_tags: config.process_tags.clone(), @@ -277,8 +276,8 @@ pub(crate) fn get_or_create_concentrator( ) }; - let process_tags = session.process_tags.lock_or_panic().clone(); - let instance_id = session.get_runtime(&runtime_id.to_string()).instance_id; + let process_tags = session.process_tags_with_svc_source(); + let instance_id = InstanceId::new(session.session_id.clone(), auxiliary_runtime_id); let session_config_closure = || { session .session_config @@ -290,8 +289,8 @@ pub(crate) fn get_or_create_concentrator( Config::default() }) }; - let telemetry = { - let telemetry_mutex = telemetry_clients.get_or_create( + let telemetry = + crate::service::telemetry::TelemetryCachedClient::spawn_metrics_logs_worker( &service_name, env, &instance_id, @@ -299,15 +298,22 @@ pub(crate) fn get_or_create_concentrator( session_config_closure, process_tags, ); - let worker = telemetry_mutex.lock_or_panic().worker.clone(); - Some(worker) - }; + let telemetry = Some(telemetry); + let exporter = Arc::new(make_exporter( + &concentrator, + &meta, + &config.endpoint, + config.flush_interval, + telemetry.clone(), + )); let state = Arc::new(SpanConcentratorState { concentrator, endpoint: config.endpoint.clone(), + #[cfg(test)] meta, telemetry, + exporter, }); guard.insert(map_key.clone(), state.clone()); let weak = Arc::downgrade(concentrators); @@ -326,30 +332,16 @@ pub(crate) fn get_or_create_concentrator( } /// Immediately flush all active SHM span concentrators and send the results to the agent. -pub async fn flush_all_stats_now( - state: &Arc>>>, -) { - let states: Vec<_> = { - let guard = state.lock_or_panic(); - guard - .values() - .map(|s| { - ( - make_exporter(s, s.endpoint.clone(), Duration::from_secs(10)), - s.endpoint.clone(), - ) - }) - .collect() - }; +pub async fn flush_all_stats_now(states: &[Arc]) { debug!( "Flushing all stats now: {} different exporters", states.len() ); - join_all(states.iter().map(|(exporter, endpoint)| { - exporter.send(false).inspect_err(move |e| { + join_all(states.iter().map(|state| { + state.exporter.send(false).inspect_err(move |e| { warn!( "flush_all_stats_now: failed to send stats to {:?}: {}", - endpoint, e + state.endpoint, e ); }) })) diff --git a/datadog-sidecar/src/service/telemetry.rs b/datadog-sidecar/src/service/telemetry.rs index b84be9bae6..bcaaad84ae 100644 --- a/datadog-sidecar/src/service/telemetry.rs +++ b/datadog-sidecar/src/service/telemetry.rs @@ -5,7 +5,7 @@ use crate::service::{InstanceId, RuntimeMetadata, SidecarAction, SidecarServer}; use anyhow::{anyhow, Result}; use libdd_common::MutexExt; use std::sync::OnceLock; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot, watch}; use tracing::{debug, info, warn}; use crate::primary_sidecar_identifier; @@ -15,6 +15,7 @@ use datadog_ipc::one_way_shared_memory::OneWayShmWriter; use datadog_ipc::platform::NamedShmHandle; use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet, VecDeque}; +use std::convert::Infallible; use std::ffi::CString; use std::hash::{Hash, Hasher}; use std::path::PathBuf; @@ -35,7 +36,7 @@ use std::time::SystemTime; use libdd_telemetry::config::Config; use libdd_telemetry::data::{self, Integration}; use libdd_telemetry::metrics::{ContextKey, MetricContext}; -use libdd_telemetry::worker::{TelemetryActions, TelemetryWorkerFlavor}; +use libdd_telemetry::worker::{LifecycleAction, TelemetryActions, TelemetryWorkerFlavor}; /// Sidecar's telemetry worker is native-only, so its handle is pinned to /// [`NativeCapabilities`]. @@ -44,34 +45,6 @@ use manual_future::ManualFuture; use serde_with::{serde_as, VecSkipError}; use tokio::time::{sleep, sleep_until, Instant as TokioInstant}; -fn register_metric( - worker: &TelemetryWorkerHandle, - telemetry_metrics: &mut HashMap, - metric: MetricContext, -) { - if !telemetry_metrics.contains_key(&metric.name) { - telemetry_metrics.insert( - metric.name.clone(), - worker.register_metric_context( - metric.name, - metric.tags, - metric.metric_type, - metric.common, - metric.namespace, - ), - ); - } -} - -fn to_telemetry_point( - telemetry_metrics: &HashMap, - (name, value, tags): (String, f64, Vec), -) -> Option { - telemetry_metrics - .get(&name) - .map(|context_key| TelemetryActions::AddPoint((value, *context_key, tags))) -} - #[derive(Debug)] pub struct InternalTelemetryActions { pub instance_id: InstanceId, @@ -87,22 +60,377 @@ pub enum InternalTelemetryAction { AddMetricPoint((f64, String, Vec)), } +#[derive(Clone, Copy)] +struct DirectTelemetryLifecycleState { + generation: u64, + active: bool, + retiring: u32, +} + +#[derive(Clone, Default)] +pub(crate) struct DirectTelemetryLifecycleRegistry { + inner: Arc>, +} + +#[derive(Default)] +struct DirectTelemetryLifecycleRegistryInner { + states: HashMap, + retired_sessions: HashSet, + retiring_sessions: HashMap, + next_generation: u64, +} + +impl DirectTelemetryLifecycleRegistryInner { + fn allocate_generation(&mut self) -> u64 { + assert!( + self.next_generation < u64::MAX, + "direct telemetry lifecycle generation exhausted" + ); + self.next_generation += 1; + self.next_generation + } +} + +impl DirectTelemetryLifecycleRegistry { + fn activate_inner(inner: &mut DirectTelemetryLifecycleRegistryInner, instance_id: &InstanceId) { + if inner + .retiring_sessions + .contains_key(&instance_id.session_id) + { + inner + .states + .entry(instance_id.clone()) + .or_insert(DirectTelemetryLifecycleState { + generation: 0, + active: false, + retiring: 1, + }); + return; + } + inner.retired_sessions.remove(&instance_id.session_id); + if let Some(state) = inner.states.get(instance_id) { + if state.active || state.retiring > 0 { + return; + } + } + let generation = inner.allocate_generation(); + inner.states.insert( + instance_id.clone(), + DirectTelemetryLifecycleState { + generation, + active: true, + retiring: 0, + }, + ); + } + + pub(crate) fn activate_with( + &self, + instance_id: &InstanceId, + activate: impl FnOnce() -> T, + ) -> T { + let mut inner = self.inner.lock_or_panic(); + let result = activate(); + Self::activate_inner(&mut inner, instance_id); + result + } + + pub(crate) fn begin_retire_runtimes( + &self, + instances: &HashSet, + ) -> HashMap { + let mut inner = self.inner.lock_or_panic(); + let mut retirements = HashMap::with_capacity(instances.len()); + for instance_id in instances { + let state = inner + .states + .entry(instance_id.clone()) + .and_modify(|state| { + assert!( + state.retiring < u32::MAX, + "direct telemetry runtime retirement count exhausted" + ); + state.retiring += 1; + }) + .or_insert(DirectTelemetryLifecycleState { + generation: 0, + active: false, + retiring: 1, + }); + retirements.insert(instance_id.clone(), state.generation); + } + retirements + } + + pub(crate) fn retire_runtimes( + &self, + retirements: &HashMap, + ) -> HashSet { + let mut inner = self.inner.lock_or_panic(); + let mut retired = HashSet::with_capacity(retirements.len()); + for (instance_id, generation) in retirements { + if let Some(state) = inner.states.get_mut(instance_id) { + if state.generation == *generation { + state.active = false; + retired.insert(instance_id.clone()); + } + } + } + retired + } + + pub(crate) fn finish_retire_runtimes( + &self, + retirements: &HashMap, + finish: impl FnOnce(&HashSet) -> T, + ) -> T { + let mut inner = self.inner.lock_or_panic(); + let mut ready = HashSet::new(); + for (instance_id, generation) in retirements { + if let Some(state) = inner.states.get_mut(instance_id) { + if state.generation == *generation && state.retiring > 0 { + state.retiring -= 1; + if state.retiring == 0 { + ready.insert(instance_id.clone()); + } + } + } + } + finish(&ready) + } + + pub(crate) fn begin_retire_session(&self, session_id: &str) { + let mut inner = self.inner.lock_or_panic(); + let count = inner + .retiring_sessions + .entry(session_id.to_string()) + .or_default(); + assert!( + *count < u32::MAX, + "direct telemetry session retirement count exhausted" + ); + *count += 1; + } + + pub(crate) fn retire_session(&self, session_id: &str) { + let mut inner = self.inner.lock_or_panic(); + inner + .states + .retain(|instance_id, _| instance_id.session_id != session_id); + inner.retired_sessions.insert(session_id.to_string()); + } + + pub(crate) fn finish_retire_session( + &self, + session_id: &str, + finish: impl FnOnce() -> T, + ) -> Option { + let mut inner = self.inner.lock_or_panic(); + let count = inner.retiring_sessions.get_mut(session_id)?; + *count -= 1; + if *count > 0 { + return None; + } + let result = finish(); + inner + .states + .retain(|instance_id, _| instance_id.session_id != session_id); + inner.retiring_sessions.remove(session_id); + Some(result) + } + + fn state(&self, instance_id: &InstanceId) -> Option { + let inner = self.inner.lock_or_panic(); + inner.states.get(instance_id).copied().or_else(|| { + inner + .retired_sessions + .contains(&instance_id.session_id) + .then_some(DirectTelemetryLifecycleState { + generation: 0, + active: false, + retiring: 0, + }) + }) + } + + fn generation(&self, instance_id: &InstanceId) -> Option { + self.state(instance_id).map(|state| state.generation) + } + + #[cfg(test)] + pub(crate) fn is_retiring(&self, instance_id: &InstanceId) -> bool { + let inner = self.inner.lock_or_panic(); + inner + .states + .get(instance_id) + .is_some_and(|state| state.retiring > 0) + || inner + .retiring_sessions + .contains_key(&instance_id.session_id) + } +} + +pub(crate) struct DirectTelemetryActions { + actions: InternalTelemetryActions, + generation: Option, +} + +#[derive(Clone, Debug)] +pub(crate) enum DirectTelemetryRetirement { + Runtimes(HashMap), + Session(String), +} + +pub(crate) enum DirectTelemetryMessage { + Actions(DirectTelemetryActions), + Retire { + scope: DirectTelemetryRetirement, + acknowledgement: oneshot::Sender<()>, + }, + #[cfg(test)] + Barrier(oneshot::Sender<()>), +} + +#[derive(Clone)] +pub struct TelemetryActionSender { + sender: mpsc::Sender, + lifecycles: DirectTelemetryLifecycleRegistry, +} + +impl TelemetryActionSender { + /// The error retains the complete unsent batch so callers can retry without data loss. + #[allow(clippy::result_large_err)] + pub fn try_send( + &self, + actions: InternalTelemetryActions, + ) -> std::result::Result<(), mpsc::error::TrySendError> { + let generation = self.lifecycles.generation(&actions.instance_id); + self.sender + .try_send(DirectTelemetryMessage::Actions(DirectTelemetryActions { + actions, + generation, + })) + .map_err(|error| match error { + mpsc::error::TrySendError::Full(DirectTelemetryMessage::Actions(actions)) => { + mpsc::error::TrySendError::Full(actions.actions) + } + mpsc::error::TrySendError::Closed(DirectTelemetryMessage::Actions(actions)) => { + mpsc::error::TrySendError::Closed(actions.actions) + } + _ => unreachable!("try_send only submits direct telemetry action messages"), + }) + } + + #[cfg(test)] + pub(crate) async fn send_actions( + &self, + actions: InternalTelemetryActions, + ) -> std::result::Result<(), mpsc::error::SendError> { + let generation = self.lifecycles.generation(&actions.instance_id); + self.sender + .send(DirectTelemetryMessage::Actions(DirectTelemetryActions { + actions, + generation, + })) + .await + .map_err(|error| match error.0 { + DirectTelemetryMessage::Actions(actions) => mpsc::error::SendError(actions.actions), + _ => unreachable!("send_actions only submits direct telemetry action messages"), + }) + } + + pub(crate) async fn retire(&self, scope: DirectTelemetryRetirement) -> Result<()> { + let (acknowledgement, acknowledged) = oneshot::channel(); + self.sender + .send(DirectTelemetryMessage::Retire { + scope, + acknowledgement, + }) + .await + .map_err(|_| anyhow!("direct telemetry receiver is unavailable"))?; + acknowledged + .await + .map_err(|_| anyhow!("direct telemetry cleanup acknowledgement was dropped")) + } + + #[cfg(test)] + pub(crate) async fn barrier(&self) -> Result<()> { + let (sender, receiver) = oneshot::channel(); + self.sender + .send(DirectTelemetryMessage::Barrier(sender)) + .await + .map_err(|_| anyhow!("direct telemetry receiver is unavailable"))?; + receiver + .await + .map_err(|_| anyhow!("direct telemetry receiver barrier was dropped")) + } +} + +#[cfg(test)] +pub(crate) fn direct_telemetry_channel( + sidecar: &SidecarServer, +) -> ( + TelemetryActionSender, + mpsc::Receiver, +) { + let (sender, receiver) = mpsc::channel(1000); + ( + TelemetryActionSender { + sender, + lifecycles: sidecar.direct_telemetry_lifecycles.clone(), + }, + receiver, + ) +} + pub(crate) async fn telemetry_action_receiver_task( sidecar: SidecarServer, - mut rx: mpsc::Receiver, + mut rx: mpsc::Receiver, ) { info!("Starting telemetry action receiver task..."); let mut pending: Vec = Vec::new(); - while let Some(batch) = next_entry(&mut pending, &mut rx).await { - let Some(telemetry_client) = batch.get_client(&sidecar) else { - batch.defer_or_drop(&mut pending); + while let Some(entry) = next_entry(&mut pending, &mut rx).await { + let ReceiverEntry::Batch(batch) = entry else { + match entry { + ReceiverEntry::Retire { + scope, + acknowledgement, + } => { + match &scope { + DirectTelemetryRetirement::Runtimes(retirements) => { + let retired = sidecar + .direct_telemetry_lifecycles + .retire_runtimes(retirements); + pending.retain(|batch| match retirements.get(&batch.key.0) { + Some(generation) => batch.key.3.is_some_and(|batch_generation| { + batch_generation != *generation + }), + None => true, + }); + sidecar.metrics_logs_clients.remove_runtimes(&retired); + } + DirectTelemetryRetirement::Session(session_id) => { + sidecar + .direct_telemetry_lifecycles + .retire_session(session_id); + pending.retain(|batch| batch.key.0.session_id != *session_id); + sidecar.metrics_logs_clients.remove_session(session_id); + } + } + let _ = acknowledgement.send(()); + } + #[cfg(test)] + ReceiverEntry::Barrier(acknowledgement) => { + let _ = acknowledgement.send(()); + } + ReceiverEntry::Batch(_) => unreachable!(), + } continue; }; - - let client = telemetry_client.lock_or_panic().worker.clone(); - - batch.deliver(&telemetry_client, &client).await; + if let Err(batch) = batch.deliver(&sidecar).await { + batch.defer_or_drop(&mut pending); + } } let total_pending: usize = pending.iter().map(|s| s.actions.len()).sum(); @@ -117,11 +445,11 @@ pub(crate) async fn telemetry_action_receiver_task( async fn next_entry( pending: &mut Vec, - rx: &mut mpsc::Receiver, -) -> Option { + rx: &mut mpsc::Receiver, +) -> Option { loop { if pending.is_empty() { - return rx.recv().await.map(TelemetryBatch::Fresh); + return rx.recv().await.map(ReceiverEntry::from); } // we have batches to retry @@ -138,33 +466,80 @@ async fn next_entry( tokio::select! { biased; _ = sleep_until(deadline) => { - return Some(TelemetryBatch::Deferred(pending.swap_remove(min_pos))); + return Some(ReceiverEntry::Batch(TelemetryBatch::Deferred( + pending.swap_remove(min_pos), + ))); } result = rx.recv() => match result { - Some(batch) => { - let key = (batch.service_name.as_str(), batch.env_name.as_str()); - if let Some(deferred) = pending.iter_mut().find(|s| s.key.0 == key.0 && s.key.1 == key.1) { + Some(DirectTelemetryMessage::Actions(batch)) => { + let key = ( + &batch.actions.instance_id, + batch.actions.service_name.as_str(), + batch.actions.env_name.as_str(), + batch.generation, + ); + if let Some(deferred) = pending.iter_mut().find(|batch| { + batch.key.0 == *key.0 + && batch.key.1 == key.1 + && batch.key.2 == key.2 + && batch.key.3 == key.3 + }) { deferred.actions.push_back(batch); } else { - return Some(TelemetryBatch::Fresh(batch)); + return Some(ReceiverEntry::Batch(TelemetryBatch::Fresh(batch))); } } + Some(message) => return Some(ReceiverEntry::from(message)), None => return None, }, } } } +enum ReceiverEntry { + Batch(TelemetryBatch), + Retire { + scope: DirectTelemetryRetirement, + acknowledgement: oneshot::Sender<()>, + }, + #[cfg(test)] + Barrier(oneshot::Sender<()>), +} + +impl From for ReceiverEntry { + fn from(message: DirectTelemetryMessage) -> Self { + match message { + DirectTelemetryMessage::Actions(actions) => Self::Batch(TelemetryBatch::Fresh(actions)), + DirectTelemetryMessage::Retire { + scope, + acknowledgement, + } => Self::Retire { + scope, + acknowledgement, + }, + #[cfg(test)] + DirectTelemetryMessage::Barrier(acknowledgement) => Self::Barrier(acknowledgement), + } + } +} + async fn deliver_batch( actions: Vec, - telemetry_client: &Arc>, - client: &TelemetryWorkerHandle, + sidecar: &SidecarServer, + instance_id: &InstanceId, + service: &str, + env: &str, + active_client: &mut BatchDirectTelemetryClient, ) { for it_action in actions { match it_action { InternalTelemetryAction::TelemetryAction(action) => { + let Some(client) = active_client.get(sidecar, instance_id, service, env) else { + warn!("Telemetry client unavailable during delivery for {service}/{env}"); + continue; + }; let action_str = format!("{action:?}"); - match client.send_msg(action).await { + match client.worker.send_msg(action).await { Ok(_) => debug!("Sent telemetry action to TelemetryWorker: {action_str}"), Err(e) => warn!( "Failed to send telemetry action {action_str} to TelemetryWorker: {e}" @@ -172,17 +547,46 @@ async fn deliver_batch( } } InternalTelemetryAction::RegisterTelemetryMetric(metric) => { - debug!("Registered telemetry metric: {metric:?}"); - telemetry_client.lock_or_panic().register_metric(metric); + let metric_name = metric.name.clone(); + let outcome = sidecar.metrics_logs_clients.register_metric_with_outcome( + instance_id, + service, + env, + metric, + ); + if outcome == MetricRegistrationOutcome::Changed { + active_client.invalidate(); + } + match outcome { + MetricRegistrationOutcome::RejectedCapacity { limit } => warn!( + "Rejected telemetry metric registration: session={} service={} env={} \ + metric={} capacity={limit}", + instance_id.session_id, service, env, metric_name + ), + outcome => debug!( + "Registered telemetry metric: session={} service={} env={} metric={} \ + outcome={outcome:?}", + instance_id.session_id, service, env, metric_name + ), + } } InternalTelemetryAction::AddMetricPoint((value, name, tags)) => { let metric_name = name.clone(); - let point = telemetry_client + let Some(client) = active_client.get(sidecar, instance_id, service, env) else { + warn!( + "Telemetry client unavailable for metric point {metric_name} in \ + {service}/{env}" + ); + continue; + }; + let point = client + .client .lock_or_panic() - .to_telemetry_point((name, value, tags)); + .as_ref() + .and_then(|t| t.to_telemetry_point((name, value, tags))); match point { Some(p) => { - if let Err(e) = client.send_msg(p).await { + if let Err(e) = client.worker.send_msg(p).await { warn!("Failed to send telemetry point to TelemetryWorker: {e}"); } } @@ -195,32 +599,111 @@ async fn deliver_batch( } } +struct ActiveDirectTelemetryClient { + client: Arc>>, + worker: TelemetryWorkerHandle, +} + +enum BatchDirectTelemetryClient { + Active(ActiveDirectTelemetryClient), + RefreshRequired, + Unavailable, +} + +impl BatchDirectTelemetryClient { + fn invalidate(&mut self) { + *self = Self::RefreshRequired; + } + + fn get( + &mut self, + sidecar: &SidecarServer, + instance_id: &InstanceId, + service: &str, + env: &str, + ) -> Option<&ActiveDirectTelemetryClient> { + if matches!(self, Self::RefreshRequired) { + *self = get_active_direct_worker(sidecar, instance_id, service, env) + .map(Self::Active) + .unwrap_or(Self::Unavailable); + } + match self { + Self::Active(client) => Some(client), + Self::RefreshRequired | Self::Unavailable => None, + } + } +} + +fn get_active_direct_worker( + sidecar: &SidecarServer, + instance_id: &InstanceId, + service: &str, + env: &str, +) -> Option { + let telemetry_client = get_telemetry_client(sidecar, instance_id, service, env)?; + let worker = telemetry_client + .lock_or_panic() + .as_ref() + .filter(|client| !client.is_stopping()) + .map(|client| client.worker.clone())?; + Some(ActiveDirectTelemetryClient { + client: telemetry_client, + worker, + }) +} + enum TelemetryBatch { - Fresh(InternalTelemetryActions), + Fresh(DirectTelemetryActions), Deferred(PerClientTelemetryBatch), } +enum DirectTelemetryLifecycle { + Ready, + Retryable, + Retired, +} + impl TelemetryBatch { - fn get_client(&self, sidecar: &SidecarServer) -> Option>> { + fn key(&self) -> (&InstanceId, &str, &str) { match self { - TelemetryBatch::Fresh(a) => { - get_telemetry_client(sidecar, &a.instance_id, &a.service_name, &a.env_name) - } - TelemetryBatch::Deferred(d) => { - let (service_name, env_name) = &d.key; - let mut tried_sessions = HashSet::new(); - for b in &d.actions { - if tried_sessions.insert(b.instance_id.session_id.as_str()) { - // repeated calls to get_existing_client could be avoided - if let Some(client) = - get_telemetry_client(sidecar, &b.instance_id, service_name, env_name) - { - return Some(client); - } - } - } - None + TelemetryBatch::Fresh(actions) => ( + &actions.actions.instance_id, + &actions.actions.service_name, + &actions.actions.env_name, + ), + TelemetryBatch::Deferred(deferred) => { + (&deferred.key.0, &deferred.key.1, &deferred.key.2) + } + } + } + + fn generation(&self) -> Option { + match self { + TelemetryBatch::Fresh(actions) => actions.generation, + TelemetryBatch::Deferred(deferred) => deferred.key.3, + } + } + + fn lifecycle(&self, sidecar: &SidecarServer) -> DirectTelemetryLifecycle { + let (instance_id, _, _) = self.key(); + let current = sidecar.direct_telemetry_lifecycles.state(instance_id); + if let Some(generation) = self.generation() { + if !current.is_some_and(|state| state.active && state.generation == generation) { + return DirectTelemetryLifecycle::Retired; } + } else if current.is_some_and(|state| !state.active) { + return DirectTelemetryLifecycle::Retired; + } + let Some(session) = sidecar.find_session(&instance_id.session_id) else { + return DirectTelemetryLifecycle::Retryable; + }; + if session.find_runtime(&instance_id.runtime_id).is_none() { + return DirectTelemetryLifecycle::Retryable; + } + if session.session_config.lock_or_panic().is_none() { + DirectTelemetryLifecycle::Retryable + } else { + DirectTelemetryLifecycle::Ready } } @@ -231,16 +714,21 @@ impl TelemetryBatch { match self { TelemetryBatch::Fresh(actions) => { info!( - "No telemetry session config for {}/{}, \ + "Telemetry client not ready for {}/{}, \ retrying in {}ms ({} left)", - actions.service_name, - actions.env_name, + actions.actions.service_name, + actions.actions.env_name, Self::RETRY_DELAY.as_millis(), Self::MAX_ATTEMPTS - 1, ); let next_at = TokioInstant::now() + Self::RETRY_DELAY; pending.push(PerClientTelemetryBatch { - key: (actions.service_name.clone(), actions.env_name.clone()), + key: ( + actions.actions.instance_id.clone(), + actions.actions.service_name.clone(), + actions.actions.env_name.clone(), + actions.generation, + ), actions: VecDeque::from([actions]), attempts_left: Self::MAX_ATTEMPTS - 1, next_attempt_at: next_at, @@ -248,11 +736,11 @@ impl TelemetryBatch { } TelemetryBatch::Deferred(deferred) => { debug_assert!(!deferred.actions.is_empty()); - let (service_name, env_name) = &deferred.key; + let (_, service_name, env_name, _) = &deferred.key; let remaining = deferred.attempts_left - 1; if remaining > 0 { info!( - "No telemetry session config for {service_name}/{env_name}, \ + "Telemetry client not ready for {service_name}/{env_name}, \ retrying in {}ms ({remaining} left)", Self::RETRY_DELAY.as_millis(), ); @@ -263,10 +751,14 @@ impl TelemetryBatch { next_attempt_at: TokioInstant::now() + Self::RETRY_DELAY, }); } else { - let count: usize = deferred.actions.iter().map(|b| b.actions.len()).sum(); + let count: usize = deferred + .actions + .iter() + .map(|batch| batch.actions.actions.len()) + .sum(); warn!( "Dropping {count} telemetry actions for {service_name}/{env_name}: \ - session_config never arrived after {} attempts", + telemetry client never became ready after {} attempts", Self::MAX_ATTEMPTS, ); } @@ -274,28 +766,56 @@ impl TelemetryBatch { } } - async fn deliver( - self, - telemetry_client: &Arc>, - client: &TelemetryWorkerHandle, - ) { + async fn deliver(self, sidecar: &SidecarServer) -> std::result::Result<(), Self> { + match self.lifecycle(sidecar) { + DirectTelemetryLifecycle::Ready => {} + DirectTelemetryLifecycle::Retryable => return Err(self), + DirectTelemetryLifecycle::Retired => { + let (instance_id, service, env) = self.key(); + debug!( + "Dropping direct telemetry batch for retired lifecycle \ + {instance_id:?}/{service}/{env}" + ); + return Ok(()); + } + } + let mut active_client = BatchDirectTelemetryClient::RefreshRequired; match self { TelemetryBatch::Fresh(actions) => { - deliver_batch(actions.actions, telemetry_client, client).await; + let actions = actions.actions; + deliver_batch( + actions.actions, + sidecar, + &actions.instance_id, + &actions.service_name, + &actions.env_name, + &mut active_client, + ) + .await; } TelemetryBatch::Deferred(deferred) => { debug_assert!(!deferred.actions.is_empty()); for batch in deferred.actions { - deliver_batch(batch.actions, telemetry_client, client).await; + let batch = batch.actions; + deliver_batch( + batch.actions, + sidecar, + &batch.instance_id, + &batch.service_name, + &batch.env_name, + &mut active_client, + ) + .await; } } } + Ok(()) } } struct PerClientTelemetryBatch { - key: TelemetryCachedClientKey, - actions: VecDeque, // invariant: non-empty + key: (InstanceId, ServiceString, EnvString, Option), + actions: VecDeque, // invariant: non-empty attempts_left: u8, next_attempt_at: TokioInstant, } @@ -307,7 +827,7 @@ static COMPOSER_CACHE: LazyLock> = static LAST_CACHE_CLEAN: AtomicU64 = AtomicU64::new(0); -static TELEMETRY_ACTION_SENDER: OnceLock> = OnceLock::new(); +static TELEMETRY_ACTION_SENDER: OnceLock = OnceLock::new(); #[serde_as] #[derive(Deserialize)] @@ -322,14 +842,22 @@ pub struct TelemetryCachedEntry { } #[derive(Default)] -pub(crate) struct InitialTelemetryData { +pub struct InitialTelemetryData { configurations: Vec, dependencies: Vec, integrations: Vec, } impl InitialTelemetryData { - pub(crate) fn from_actions(actions: &[SidecarAction]) -> Self { + pub fn from_actions(actions: &[SidecarAction]) -> Self { + Self::from_action_refs(actions.iter()) + } + + fn from_pending_actions(actions: &[PendingApplicationAction]) -> Self { + Self::from_action_refs(actions.iter().map(|pending_action| &pending_action.action)) + } + + fn from_action_refs<'a>(actions: impl Iterator) -> Self { let mut initial = Self::default(); for action in actions { match action { @@ -347,14 +875,137 @@ impl InitialTelemetryData { } initial } + + pub(crate) fn contains_seeded_action(action: &SidecarAction) -> bool { + matches!( + action, + SidecarAction::Telemetry( + TelemetryActions::AddConfig(_) + | TelemetryActions::AddDependency(_) + | TelemetryActions::AddIntegration(_) + ) + ) + } +} + +struct PendingTelemetryActions { + last_used: Instant, + actions: Vec, +} + +type PendingTelemetryKey = (ServiceString, EnvString); + +#[derive(Debug)] +pub(crate) struct PendingApplicationAction { + pub(crate) origin: InstanceId, + pub(crate) action: SidecarAction, + pub(crate) metric_registration: Option, +} + +impl PendingApplicationAction { + pub(crate) fn from_actions( + origin: &InstanceId, + actions: Vec, + metric_registrations: &HashMap, + ) -> Vec { + actions + .into_iter() + .map(|action| { + let metric_registration = match &action { + SidecarAction::AddTelemetryMetricPoint((name, _, _)) => { + metric_registrations.get(name).cloned() + } + _ => None, + }; + Self { + origin: origin.clone(), + action, + metric_registration, + } + }) + .collect() + } + + pub(crate) fn promotes_lifecycle(&self) -> bool { + matches!( + self.action, + SidecarAction::Telemetry(TelemetryActions::AddConfig(_)) + | SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)) + ) + } + + fn is_stop(&self) -> bool { + matches!( + self.action, + SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)) + ) + } + + pub(crate) fn split_after_first_stop(actions: &mut Vec) -> Vec { + actions + .iter() + .position(Self::is_stop) + .filter(|stop_index| *stop_index + 1 < actions.len()) + .map(|stop_index| actions.split_off(stop_index + 1)) + .unwrap_or_default() + } +} + +pub(crate) enum ApplicationTelemetryDispatch { + Pending, + Handoff { + completion: watch::Receiver, + actions: Vec, + }, + Ready { + client: Arc>>, + actions: Vec, + created: bool, + remove_client: bool, + }, +} + +enum ApplicationShmState { + NotRequired, + Ready(OneWayShmWriter), + RetryAt { path: CString, deadline: Instant }, } pub struct TelemetryCachedClient { pub worker: TelemetryWorkerHandle, - pub shm_writer: OneWayShmWriter, + pub(crate) worker_join: Option>, + pub(crate) terminal_handoff: Option>, + shm_state: ApplicationShmState, pub telemetry_metrics: HashMap, pub handle: Option>, pub shared: TelemetryCachedClientShmData, + stopping: bool, +} + +pub(crate) struct TelemetryWorkerMetadata<'a> { + service: &'a str, + env: &'a str, + instance_id: &'a InstanceId, + runtime_meta: &'a RuntimeMetadata, + process_tags: Vec, +} + +impl<'a> TelemetryWorkerMetadata<'a> { + pub(crate) fn new( + service: &'a str, + env: &'a str, + instance_id: &'a InstanceId, + runtime_meta: &'a RuntimeMetadata, + process_tags: Vec, + ) -> Self { + Self { + service, + env, + instance_id, + runtime_meta, + process_tags, + } + } } #[derive(Deserialize, Serialize)] @@ -377,25 +1028,20 @@ impl Default for TelemetryCachedClientShmData { } impl TelemetryCachedClient { - fn worker_builder( - service: &str, - env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, - process_tags: Vec, - ) -> TelemetryWorkerBuilder { + fn worker_builder(metadata: &TelemetryWorkerMetadata<'_>) -> TelemetryWorkerBuilder { let mut builder = TelemetryWorkerBuilder::new_fetch_host( - service.to_string(), - runtime_meta.language_name.to_string(), - runtime_meta.language_version.to_string(), - runtime_meta.tracer_version.to_string(), + metadata.service.to_string(), + metadata.runtime_meta.language_name.to_string(), + metadata.runtime_meta.language_version.to_string(), + metadata.runtime_meta.tracer_version.to_string(), ); - builder.runtime_id = Some(instance_id.runtime_id.clone()); + builder.runtime_id = Some(metadata.instance_id.runtime_id.clone()); - builder.application.env = Some(env.to_string()); - builder.application.process_tags = (!process_tags.is_empty()).then(|| { - process_tags + builder.application.env = Some(metadata.env.to_string()); + builder.application.process_tags = (!metadata.process_tags.is_empty()).then(|| { + metadata + .process_tags .iter() .map(|tag| tag.to_string()) .collect::>() @@ -405,79 +1051,237 @@ impl TelemetryCachedClient { } fn new( - service: &str, - env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, + metadata: TelemetryWorkerMetadata<'_>, get_config: impl FnOnce() -> Config, - process_tags: Vec, initial: InitialTelemetryData, - ) -> Self { - let mut builder = - Self::worker_builder(service, env, instance_id, runtime_meta, process_tags); - let config = get_config(); - builder.config = config.clone(); + ) -> Result { + Self::new_with_shm_factory_at(metadata, get_config, initial, Instant::now(), |path| { + OneWayShmWriter::::new(path.clone()) + }) + } + + fn new_with_shm_factory_at( + metadata: TelemetryWorkerMetadata<'_>, + get_config: impl FnOnce() -> Config, + initial: InitialTelemetryData, + now: Instant, + create: impl FnOnce(&CString) -> std::io::Result>, + ) -> Result { + let mut builder = Self::worker_builder(&metadata); + builder.config = get_config(); builder.configurations.extend(initial.configurations); builder.dependencies.extend(initial.dependencies); builder.integrations.extend(initial.integrations); - let (handle, _join) = builder.spawn(); - info!("spawned telemetry worker {config:?}"); + let (handle, worker_join) = builder.spawn(); + info!("spawned telemetry worker"); + handle.send_start()?; - if let Err(error) = handle.send_start() { - warn!("Failed to start telemetry worker: {error}"); - } + let path = path_for_telemetry(metadata.service, metadata.env); + let shm_state = match create(&path) { + Ok(writer) => ApplicationShmState::Ready(writer), + Err(error) => { + warn!("Failed to create telemetry shared-memory writer: {error:?}"); + ApplicationShmState::RetryAt { + path, + deadline: now + Duration::from_secs(60), + } + } + }; - Self { + Ok(Self { worker: handle, - shm_writer: { - #[allow(clippy::unwrap_used)] - OneWayShmWriter::::new(path_for_telemetry(service, env)).unwrap() - }, + worker_join: Some(worker_join), + terminal_handoff: None, + shm_state, shared: TelemetryCachedClientShmData::default(), telemetry_metrics: Default::default(), handle: None, - } - } - - pub fn write_shm_file(&self) { - if let Ok(buf) = bincode::serialize(&self.shared) { - self.shm_writer.write(&buf); - } else { - warn!("Failed to serialize telemetry data for shared memory"); - } - } - - pub fn register_metric(&mut self, metric: MetricContext) { - register_metric(&self.worker, &mut self.telemetry_metrics, metric); + stopping: false, + }) } - pub fn to_telemetry_point(&self, point: (String, f64, Vec)) -> Option { - to_telemetry_point(&self.telemetry_metrics, point) + #[cfg(test)] + pub(crate) fn new_with_shm_factory( + metadata: TelemetryWorkerMetadata<'_>, + get_config: impl FnOnce() -> Config, + initial: InitialTelemetryData, + now: Instant, + create: impl FnOnce(&CString) -> std::io::Result>, + ) -> Result { + Self::new_with_shm_factory_at(metadata, get_config, initial, now, create) } - pub fn process_actions( + pub(crate) fn spawn_metrics_logs_worker( + service: &str, + env: &str, + instance_id: &InstanceId, + runtime_meta: &RuntimeMetadata, + get_config: impl FnOnce() -> Config, + process_tags: Vec, + ) -> TelemetryWorkerHandle { + let metadata = + TelemetryWorkerMetadata::new(service, env, instance_id, runtime_meta, process_tags); + let mut builder = Self::worker_builder(&metadata); + builder.config = get_config(); + builder.flavor = TelemetryWorkerFlavor::MetricsLogs; + + let (handle, _join) = builder.spawn(); + info!("spawned metrics/logs telemetry worker"); + handle.send_start().ok(); + handle + } + + fn new_metrics_logs( + service: &str, + env: &str, + instance_id: &InstanceId, + runtime_meta: &RuntimeMetadata, + get_config: impl FnOnce() -> Config, + process_tags: Vec, + ) -> Self { + Self { + worker: Self::spawn_metrics_logs_worker( + service, + env, + instance_id, + runtime_meta, + get_config, + process_tags, + ), + worker_join: None, + terminal_handoff: None, + shm_state: ApplicationShmState::NotRequired, + telemetry_metrics: HashMap::new(), + handle: None, + shared: TelemetryCachedClientShmData::default(), + stopping: false, + } + } + + pub(crate) fn is_stopping(&self) -> bool { + self.stopping + } + + pub(crate) fn mark_stopping(&mut self) { + if let ApplicationShmState::Ready(shm_writer) = + std::mem::replace(&mut self.shm_state, ApplicationShmState::NotRequired) + { + shm_writer.write(&[]); + } + self.stopping = true; + } + + pub fn write_shm_file(&mut self) { + self.write_shm_file_at(Instant::now(), |path| { + OneWayShmWriter::::new(path.clone()) + }); + } + + pub(crate) fn retry_shm_file_if_due(&mut self) { + let now = Instant::now(); + if matches!( + &self.shm_state, + ApplicationShmState::RetryAt { deadline, .. } if now >= *deadline + ) { + self.write_shm_file_at(now, |path| { + OneWayShmWriter::::new(path.clone()) + }); + } + } + + #[cfg(test)] + pub(crate) fn has_ready_shm(&self) -> bool { + matches!(self.shm_state, ApplicationShmState::Ready(_)) + } + + fn write_shm_file_at( + &mut self, + now: Instant, + create: impl FnOnce(&CString) -> std::io::Result>, + ) { + let serialized = match bincode::serialize(&self.shared) { + Ok(value) => value, + Err(error) => { + warn!("Failed to serialize telemetry data for shared memory: {error}"); + return; + } + }; + + if matches!( + &self.shm_state, + ApplicationShmState::RetryAt { deadline, .. } if now >= *deadline + ) { + let ApplicationShmState::RetryAt { path, .. } = + std::mem::replace(&mut self.shm_state, ApplicationShmState::NotRequired) + else { + unreachable!(); + }; + self.shm_state = match create(&path) { + Ok(writer) => ApplicationShmState::Ready(writer), + Err(error) => { + warn!("Failed to create telemetry shared-memory writer: {error:?}"); + ApplicationShmState::RetryAt { + path, + deadline: now + Duration::from_secs(60), + } + } + }; + } + + if let ApplicationShmState::Ready(writer) = &self.shm_state { + writer.write(&serialized); + } + } + + pub fn register_metric(&mut self, metric: MetricContext) { + let name = metric.name.clone(); + let context_key = self.worker.register_metric_context( + metric.name, + metric.tags, + metric.metric_type, + metric.common, + metric.namespace, + ); + self.telemetry_metrics.insert(name, context_key); + } + + pub fn to_telemetry_point( + &self, + (name, val, tags): (String, f64, Vec), + ) -> Option { + self.telemetry_metrics + .get(&name) + .map(|context_key| TelemetryActions::AddPoint((val, *context_key, tags))) + } + + pub fn process_actions( &mut self, sidecar_actions: Vec, ) -> Vec { - let mut actions = vec![]; - for action in sidecar_actions { - match action { - SidecarAction::Telemetry(t) => actions.push(t), - SidecarAction::AddTelemetryMetricPoint(point) => { - let metric_name = point.0.clone(); - if let Some(telemetry_action) = self.to_telemetry_point(point) { - actions.push(telemetry_action); - } else { - warn!("Attempted to send telemetry point for unregistered metric: {metric_name}"); - } + sidecar_actions + .into_iter() + .filter_map(|action| self.process_action(action)) + .collect() + } + + pub(crate) fn process_action(&mut self, action: SidecarAction) -> Option { + match action { + SidecarAction::Telemetry(action) => Some(action), + SidecarAction::AddTelemetryMetricPoint(point) => { + let metric_name = point.0.clone(); + let action = self.to_telemetry_point(point); + if action.is_none() { + warn!( + "Attempted to send telemetry point for unregistered metric: {metric_name}" + ); } - SidecarAction::PhpComposerTelemetryFile(_) => {} // handled separately - SidecarAction::FfeExposureBatch(_) => {} // handled in sidecar_server - SidecarAction::FfeEvaluationMetrics { .. } => {} // handled in sidecar_server + action } + SidecarAction::PhpComposerTelemetryFile(_) + | SidecarAction::FfeExposureBatch(_) + | SidecarAction::FfeEvaluationMetrics { .. } => None, } - actions } pub async fn process_composer_paths(paths: Vec) -> Vec { @@ -553,197 +1357,160 @@ impl TelemetryCachedClient { impl Drop for TelemetryCachedClient { fn drop(&mut self) { - self.shm_writer.write(&[]); + if let ApplicationShmState::Ready(shm_writer) = + std::mem::replace(&mut self.shm_state, ApplicationShmState::NotRequired) + { + shm_writer.write(&[]); + } } } type ServiceString = String; type EnvString = String; -type TelemetryCachedClientKey = (ServiceString, EnvString); - -pub(crate) struct MetricsLogsCachedClient { - pub(crate) worker: TelemetryWorkerHandle, - pub(crate) telemetry_metrics: HashMap, +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +enum TelemetryCachedClientOwner { + Application, + Runtime(InstanceId), +} +type TelemetryCachedClientKey = (TelemetryCachedClientOwner, ServiceString, EnvString); +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct TelemetryMetricRegistrationScope { + session_id: String, + service: ServiceString, + env: EnvString, } -impl MetricsLogsCachedClient { - fn new( - service: &str, - env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, - get_config: impl FnOnce() -> Config, - process_tags: Vec, - ) -> Self { - let mut builder = TelemetryCachedClient::worker_builder( - service, - env, - instance_id, - runtime_meta, - process_tags, - ); - builder.config = get_config(); - builder.flavor = TelemetryWorkerFlavor::MetricsLogs; - - let (handle, _join) = builder.spawn(); - info!("spawned metrics/logs telemetry worker"); - if let Err(error) = handle.send_start() { - warn!("Failed to start metrics/logs telemetry worker: {error}"); - } - +impl TelemetryMetricRegistrationScope { + fn new(instance_id: &InstanceId, service: &str, env: &str) -> Self { Self { - worker: handle, - telemetry_metrics: HashMap::new(), + session_id: instance_id.session_id.clone(), + service: service.to_string(), + env: env.to_string(), } } +} - fn register_metric(&mut self, metric: MetricContext) { - register_metric(&self.worker, &mut self.telemetry_metrics, metric); - } +type TelemetryMetricRegistrations = + HashMap>; - fn to_telemetry_point(&self, point: (String, f64, Vec)) -> Option { - to_telemetry_point(&self.telemetry_metrics, point) - } -} +/// Non-terminal application batches are rejected once this many actions are pending. +/// Configuration and Stop batches are always admitted because they immediately promote and +/// complete the lifecycle transition instead of remaining buffered. +const MAX_PENDING_APPLICATION_ACTIONS: usize = 1024; -struct MetricsLogsCachedEntry { - last_used: Instant, - client: Arc>, +fn metric_contexts_match(left: &MetricContext, right: &MetricContext) -> bool { + left.name == right.name + && left.tags == right.tags + && left.metric_type == right.metric_type + && left.common == right.common + && std::mem::discriminant(&left.namespace) == std::mem::discriminant(&right.namespace) } -pub(crate) struct MetricsLogsClientSet { - inner: Arc>>, +pub struct TelemetryCachedClientSet { + inner: Arc>>, + pending: Arc>>, + pending_action_limit: usize, + /// Serializes cache replacement with the remove/retire phases of eviction. + replacement_gate: Arc>, + #[cfg(test)] + cache_lookup_count: Arc, cleanup_handle: Option>, } -impl Default for MetricsLogsClientSet { +impl Default for TelemetryCachedClientSet { fn default() -> Self { - let inner: Arc>> = + Self::with_cleanup(Duration::from_secs(1800)) + } +} + +impl TelemetryCachedClientSet { + fn with_cleanup(ttl: Duration) -> Self { + let inner: Arc>> = Arc::new(Default::default()); let clients = inner.clone(); - let cleanup_handle = tokio::spawn(async move { + let pending: Arc>> = + Arc::new(Default::default()); + let pending_actions = pending.clone(); + let replacement_gate = Arc::new(Mutex::new(())); + let cleanup_replacement_gate = replacement_gate.clone(); + + let handle = tokio::spawn(async move { loop { sleep(Duration::from_secs(60)).await; - clients + Self::evict_expired_entries( + &clients, + &cleanup_replacement_gate, + Instant::now(), + ttl, + ); + pending_actions .lock_or_panic() - .retain(|_, entry| entry.last_used.elapsed() < Duration::from_secs(1800)); + .retain(|_, actions| actions.last_used.elapsed() < ttl); } }); - Self { - inner, - cleanup_handle: Some(cleanup_handle), - } - } -} -impl Clone for MetricsLogsClientSet { - fn clone(&self) -> Self { Self { - inner: Arc::clone(&self.inner), - cleanup_handle: None, - } - } -} - -impl Drop for MetricsLogsClientSet { - fn drop(&mut self) { - if let Some(handle) = self.cleanup_handle.take() { - handle.abort(); + inner, + pending, + pending_action_limit: MAX_PENDING_APPLICATION_ACTIONS, + replacement_gate, + #[cfg(test)] + cache_lookup_count: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + cleanup_handle: Some(handle), } } -} -impl MetricsLogsClientSet { - fn get_existing_client( - &self, - service: &str, - env: &str, - ) -> Option>> { - let key = (service.to_string(), env.to_string()); - let mut clients = self.inner.lock_or_panic(); - clients.get_mut(&key).map(|entry| { - entry.last_used = Instant::now(); - entry.client.clone() - }) - } + fn evict_expired_entries( + clients: &Arc>>, + replacement_gate: &Arc>, + now: Instant, + ttl: Duration, + ) { + let _replacement_guard = replacement_gate.lock_or_panic(); + let removed = { + let mut clients = clients.lock_or_panic(); + let expired = clients + .iter() + .filter(|(_, entry)| now.saturating_duration_since(entry.last_used) >= ttl) + .map(|(key, _)| key.clone()) + .collect::>(); + expired + .into_iter() + .filter_map(|key| clients.remove(&key)) + .collect::>() + }; - pub(crate) fn get_or_create( - &self, - service: &str, - env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, - get_config: F, - process_tags: Vec, - ) -> Arc> - where - F: FnOnce() -> Config, - { - let mut clients = self.inner.lock_or_panic(); - let key = (service.to_string(), env.to_string()); - match clients.entry(key) { - Entry::Occupied(mut entry) => { - entry.get_mut().last_used = Instant::now(); - entry.get().client.clone() - } - Entry::Vacant(entry) => { - let client = Arc::new(Mutex::new(MetricsLogsCachedClient::new( - service, - env, - instance_id, - runtime_meta, - get_config, - process_tags, - ))); - entry.insert(MetricsLogsCachedEntry { - last_used: Instant::now(), - client: client.clone(), - }); - info!("Created new metrics/logs telemetry client for {service:?}/{env:?}"); - client + // The cache lock is released before retiring individual clients. The replacement gate + // remains held, so no caller can publish a replacement named-SHM owner until every old + // owner has relinquished its writer. + for entry in &removed { + if let Some(client) = entry.client.lock_or_panic().as_mut() { + client.mark_stopping(); } } + drop(removed); } - pub(crate) fn clients(&self) -> Vec>> { - self.inner - .lock_or_panic() - .values() - .map(|entry| entry.client.clone()) - .collect() + #[cfg(test)] + fn evict_expired_at(&self, now: Instant, ttl: Duration) { + Self::evict_expired_entries(&self.inner, &self.replacement_gate, now, ttl); } - pub(crate) fn workers(&self) -> Vec { - self.clients() - .into_iter() - .map(|client| client.lock_or_panic().worker.clone()) - .collect() + #[cfg(test)] + fn with_pending_action_limit(limit: usize) -> Self { + let mut clients = Self::with_cleanup(Duration::from_secs(1800)); + clients.pending_action_limit = limit; + clients } -} - -pub struct TelemetryCachedClientSet { - pub inner: Arc>>, - cleanup_handle: Option>, -} - -impl Default for TelemetryCachedClientSet { - fn default() -> Self { - let inner: Arc>> = - Arc::new(Default::default()); - let clients = inner.clone(); - let handle = tokio::spawn(async move { - loop { - sleep(Duration::from_secs(60)).await; - let mut lock = clients.lock_or_panic(); - lock.retain(|_, c| c.last_used.elapsed() < Duration::from_secs(1800)); - } + pub(crate) fn remove_pending_session(&self, session_id: &str) { + let _replacement_guard = self.replacement_gate.lock_or_panic(); + self.pending.lock_or_panic().retain(|_, pending| { + pending + .actions + .retain(|action| action.origin.session_id != session_id); + !pending.actions.is_empty() }); - - Self { - inner, - cleanup_handle: Some(handle), - } } } @@ -759,97 +1526,706 @@ impl Clone for TelemetryCachedClientSet { fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), + pending: Arc::clone(&self.pending), + pending_action_limit: self.pending_action_limit, + replacement_gate: Arc::clone(&self.replacement_gate), + #[cfg(test)] + cache_lookup_count: Arc::clone(&self.cache_lookup_count), cleanup_handle: None, } } } impl TelemetryCachedClientSet { - pub(crate) fn clients(&self) -> Vec>>> { - self.inner - .lock_or_panic() - .values() - .map(|entry| entry.client.clone()) - .collect() + #[cfg(test)] + fn get_existing_client( + &self, + service: &str, + env: &str, + ) -> Option>>> { + self.get_existing_client_with(TelemetryCachedClientOwner::Application, service, env) } - pub fn get_or_create( + fn get_existing_client_with( &self, + owner: TelemetryCachedClientOwner, service: &str, env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, - get_config: F, - process_tags: Vec, - ) -> Arc>> - where - F: FnOnce() -> Config, - { - self.get_or_create_with_initial( - service, - env, - instance_id, - runtime_meta, - get_config, - process_tags, - InitialTelemetryData::default, - ) - .0 + ) -> Option>>> { + #[cfg(test)] + self.cache_lookup_count.fetch_add(1, Ordering::Relaxed); + let key = (owner, service.to_string(), env.to_string()); + + let mut map = self.inner.lock_or_panic(); + map.get_mut(&key).map(|entry| { + entry.last_used = Instant::now(); + entry.client.clone() + }) } - #[allow(clippy::too_many_arguments)] - pub(crate) fn get_or_create_with_initial( + fn get_or_create_with( &self, + owner: TelemetryCachedClientOwner, service: &str, env: &str, - instance_id: &InstanceId, - runtime_meta: &RuntimeMetadata, - get_config: F, - process_tags: Vec, - get_initial: I, - ) -> (Arc>>, bool) - where - F: FnOnce() -> Config, - I: FnOnce() -> InitialTelemetryData, - { + create: impl FnOnce() -> TelemetryCachedClient, + ) -> Arc>> { + match self.get_or_try_create_with(owner, service, env, || Ok::<_, Infallible>(create())) { + Ok(client) => client, + Err(never) => match never {}, + } + } + + fn get_or_try_create_with( + &self, + owner: TelemetryCachedClientOwner, + service: &str, + env: &str, + create: impl FnOnce() -> std::result::Result, + ) -> std::result::Result>>, E> { + let _replacement_guard = self.replacement_gate.lock_or_panic(); let mut map = self.inner.lock_or_panic(); - let key = (service.to_string(), env.to_string()); - match map.entry(key) { - Entry::Occupied(entry) => (entry.get().client.clone(), false), + let key = (owner, service.to_string(), env.to_string()); + match map.entry(key.clone()) { + Entry::Occupied(mut entry) => { + let active = { + let client = entry.get().client.lock_or_panic(); + client.as_ref().is_some_and(|client| !client.is_stopping()) + }; + if active { + entry.get_mut().last_used = Instant::now(); + Ok(entry.get().client.clone()) + } else { + let new_client = Arc::new(Mutex::new(Some(create()?))); + entry.insert(TelemetryCachedEntry { + last_used: Instant::now(), + client: new_client.clone(), + }); + info!("Replaced stopped telemetry client for {key:?}"); + Ok(new_client) + } + } Entry::Vacant(entry) => { - let new_client = Arc::new(Mutex::new(Some(TelemetryCachedClient::new( - service, - env, - instance_id, - runtime_meta, - get_config, - process_tags, - get_initial(), - )))); + let new_client = Arc::new(Mutex::new(Some(create()?))); entry.insert(TelemetryCachedEntry { last_used: Instant::now(), client: new_client.clone(), }); - info!("Created new telemetry client for {service:?}/{env:?}"); - (new_client, true) + info!("Created new telemetry client for {key:?}"); + Ok(new_client) } } } - pub fn remove_telemetry_client(&self, service: &str, env: &str) { - let key = (service.to_string(), env.to_string()); - self.inner.lock_or_panic().remove(&key); - } -} + #[cfg(test)] + pub(crate) fn get_or_create( + &self, + metadata: TelemetryWorkerMetadata<'_>, + get_config: F, + initial: InitialTelemetryData, + ) -> Result>>> + where + F: FnOnce() -> Config, + { + let service = metadata.service; + let env = metadata.env; + self.get_or_try_create_with( + TelemetryCachedClientOwner::Application, + service, + env, + || TelemetryCachedClient::new(metadata, get_config, initial), + ) + } + + pub(crate) fn get_or_create_for_actions<'a>( + &self, + metadata: TelemetryWorkerMetadata<'a>, + actions: Vec, + get_config: impl FnOnce() -> Config, + initialize: impl FnOnce( + &Arc>>, + Vec, + ) -> bool, + ) -> ApplicationTelemetryDispatch { + self.get_or_create_for_actions_with( + metadata, + actions, + initialize, + move |metadata, initial| TelemetryCachedClient::new(metadata, get_config, initial), + ) + } + + fn get_or_create_for_actions_with<'a>( + &self, + metadata: TelemetryWorkerMetadata<'a>, + actions: Vec, + initialize: impl FnOnce( + &Arc>>, + Vec, + ) -> bool, + create_client: impl FnOnce( + TelemetryWorkerMetadata<'a>, + InitialTelemetryData, + ) -> Result, + ) -> ApplicationTelemetryDispatch { + let service = metadata.service; + let env = metadata.env; + let _replacement_guard = self.replacement_gate.lock_or_panic(); + let mut clients = self.inner.lock_or_panic(); + let key = ( + TelemetryCachedClientOwner::Application, + service.to_string(), + env.to_string(), + ); + + if let Some(entry) = clients.get_mut(&key) { + let (active, handoff) = entry + .client + .lock_or_panic() + .as_ref() + .map(|client| { + ( + !client.is_stopping(), + client + .terminal_handoff + .as_ref() + .filter(|completion| !*completion.borrow()) + .cloned(), + ) + }) + .unwrap_or((false, None)); + if active { + entry.last_used = Instant::now(); + return ApplicationTelemetryDispatch::Ready { + client: entry.client.clone(), + actions, + created: false, + remove_client: false, + }; + } + if let Some(completion) = handoff { + return ApplicationTelemetryDispatch::Handoff { + completion, + actions, + }; + } + } + clients.remove(&key); + + let pending_key = (service.to_string(), env.to_string()); + let mut pending = self.pending.lock_or_panic(); + let pending_actions = + pending + .entry(pending_key.clone()) + .or_insert_with(|| PendingTelemetryActions { + last_used: Instant::now(), + actions: Vec::new(), + }); + let incoming_promotes = actions + .iter() + .any(PendingApplicationAction::promotes_lifecycle); + let existing_payloads = pending_actions + .actions + .iter() + .filter(|pending_action| !pending_action.promotes_lifecycle()) + .count(); + if !incoming_promotes + && existing_payloads.saturating_add(actions.len()) > self.pending_action_limit + { + warn!( + "Rejecting {} application telemetry actions for {service:?}/{env:?}: \ + pending lifecycle limit {} would be exceeded", + actions.len(), + self.pending_action_limit + ); + return ApplicationTelemetryDispatch::Pending; + } + if incoming_promotes { + let mut remaining = self.pending_action_limit.saturating_sub(existing_payloads); + let mut dropped = 0; + pending_actions + .actions + .extend(actions.into_iter().filter(|action| { + if action.promotes_lifecycle() { + true + } else if remaining > 0 { + remaining -= 1; + true + } else { + dropped += 1; + false + } + })); + if dropped > 0 { + warn!( + "Dropped {dropped} non-terminal application telemetry actions for \ + {service:?}/{env:?} while preserving lifecycle promotion" + ); + } + } else { + pending_actions.actions.extend(actions); + } + pending_actions.last_used = Instant::now(); + + let should_promote = pending_actions + .actions + .iter() + .any(PendingApplicationAction::promotes_lifecycle); + if !should_promote { + return ApplicationTelemetryDispatch::Pending; + } + + let Some(pending_actions) = pending.remove(&pending_key) else { + warn!("Pending application telemetry lifecycle disappeared for {service:?}/{env:?}"); + return ApplicationTelemetryDispatch::Pending; + }; + let mut actions = pending_actions.actions; + drop(pending); + + let next_lifecycle_actions = PendingApplicationAction::split_after_first_stop(&mut actions); + let initial = InitialTelemetryData::from_pending_actions(&actions); + match create_client(metadata, initial) { + Ok(mut telemetry) => { + for pending_action in &actions { + match &pending_action.action { + SidecarAction::Telemetry(TelemetryActions::AddConfig(_)) => { + telemetry.shared.config_sent = true; + } + SidecarAction::Telemetry(TelemetryActions::AddIntegration(integration)) => { + telemetry.shared.integrations.insert(integration.clone()); + } + _ => {} + } + } + telemetry.write_shm_file(); + let client = Arc::new(Mutex::new(Some(telemetry))); + let remove_client = initialize(&client, actions); + clients.insert( + key, + TelemetryCachedEntry { + last_used: Instant::now(), + client: client.clone(), + }, + ); + info!("Created new telemetry client for {service:?}/{env:?}"); + ApplicationTelemetryDispatch::Ready { + client, + actions: next_lifecycle_actions, + created: true, + remove_client, + } + } + Err(error) => { + actions.extend(next_lifecycle_actions); + self.pending.lock_or_panic().insert( + pending_key, + PendingTelemetryActions { + last_used: Instant::now(), + actions, + }, + ); + warn!("Failed to create telemetry client for {service:?}/{env:?}: {error:?}"); + ApplicationTelemetryDispatch::Pending + } + } + } + + pub(crate) fn workers(&self) -> Vec { + self.clients() + .into_iter() + .filter_map(|client| { + client + .lock_or_panic() + .as_ref() + .map(|client| client.worker.clone()) + }) + .collect() + } + + pub(crate) fn clients(&self) -> Vec>>> { + let clients = self.inner.lock_or_panic(); + clients.values().map(|entry| entry.client.clone()).collect() + } + + fn remove_clients_matching( + &self, + predicate: impl Fn(&TelemetryCachedClientOwner, &str, &str) -> bool, + ) { + let _replacement_guard = self.replacement_gate.lock_or_panic(); + let removed = { + let mut clients = self.inner.lock_or_panic(); + let keys = clients + .keys() + .filter(|(owner, service, env)| predicate(owner, service, env)) + .cloned() + .collect::>(); + keys.into_iter() + .filter_map(|key| clients.remove(&key)) + .collect::>() + }; + for entry in &removed { + if let Some(mut client) = entry.client.lock_or_panic().take() { + client.mark_stopping(); + } + } + drop(removed); + } + + #[cfg(test)] + pub(crate) fn remove_runtime(&self, instance_id: &InstanceId) { + self.remove_clients_matching(|owner, _, _| { + matches!(owner, TelemetryCachedClientOwner::Runtime(owner_instance) if owner_instance == instance_id) + }); + } + + pub(crate) fn remove_session(&self, session_id: &str) { + self.remove_clients_matching(|owner, _, _| { + matches!(owner, TelemetryCachedClientOwner::Runtime(owner_instance) if owner_instance.session_id == session_id) + }); + } + + pub fn remove_telemetry_client( + &self, + service: &str, + env: &str, + expected: &Arc>>, + ) { + self.remove_client_with( + TelemetryCachedClientOwner::Application, + service, + env, + expected, + ); + } + + fn remove_client_with( + &self, + owner: TelemetryCachedClientOwner, + service: &str, + env: &str, + expected: &Arc>>, + ) { + let _replacement_guard = self.replacement_gate.lock_or_panic(); + let key = (owner, service.to_string(), env.to_string()); + let mut clients = self.inner.lock_or_panic(); + if clients + .get(&key) + .is_some_and(|entry| Arc::ptr_eq(&entry.client, expected)) + { + clients.remove(&key); + } + } +} + +pub(crate) struct MetricsLogsClientSet { + clients: TelemetryCachedClientSet, + registrations: Arc>, + registration_limit: usize, + #[cfg(test)] + registration_snapshot_hook: Option, +} + +#[derive(Debug, Eq, PartialEq)] +enum MetricRegistrationOutcome { + Unchanged, + Inserted, + Changed, + RejectedCapacity { limit: usize }, +} + +#[cfg(test)] +#[derive(Clone)] +pub(crate) struct MetricRegistrationSnapshotHook { + snapshot_taken: Arc, + resume_creation: Arc, + armed: Arc, +} + +#[cfg(test)] +impl MetricRegistrationSnapshotHook { + pub(crate) fn new() -> Self { + Self { + snapshot_taken: Arc::new(std::sync::Barrier::new(2)), + resume_creation: Arc::new(std::sync::Barrier::new(2)), + armed: Arc::new(std::sync::atomic::AtomicBool::new(true)), + } + } + + fn wait(&self) { + if self.armed.swap(false, Ordering::AcqRel) { + self.snapshot_taken.wait(); + self.resume_creation.wait(); + } + } + + pub(crate) fn wait_until_snapshot(&self) { + self.snapshot_taken.wait(); + } + + pub(crate) fn resume_creation(&self) { + self.resume_creation.wait(); + } +} + +impl Default for MetricsLogsClientSet { + fn default() -> Self { + Self { + clients: TelemetryCachedClientSet::default(), + registrations: Arc::new(Default::default()), + registration_limit: libdd_telemetry::worker::MAX_ITEMS, + #[cfg(test)] + registration_snapshot_hook: None, + } + } +} + +impl Clone for MetricsLogsClientSet { + fn clone(&self) -> Self { + Self { + clients: self.clients.clone(), + registrations: self.registrations.clone(), + registration_limit: self.registration_limit, + #[cfg(test)] + registration_snapshot_hook: self.registration_snapshot_hook.clone(), + } + } +} + +impl MetricsLogsClientSet { + pub(crate) fn workers(&self) -> Vec { + self.clients.workers() + } + + pub(crate) fn clients(&self) -> Vec>>> { + self.clients.clients() + } + + #[cfg(test)] + pub(crate) fn remove_runtime(&self, instance_id: &InstanceId) { + self.clients.remove_runtime(instance_id); + } + + pub(crate) fn remove_runtimes(&self, instance_ids: &HashSet) { + self.clients.remove_clients_matching(|owner, _, _| { + matches!( + owner, + TelemetryCachedClientOwner::Runtime(instance_id) + if instance_ids.contains(instance_id) + ) + }); + } + + pub(crate) fn remove_session(&self, session_id: &str) { + { + let mut registrations = self.registrations.lock_or_panic(); + registrations.retain(|scope, _| scope.session_id != session_id); + } + self.clients.remove_session(session_id); + } + + #[cfg(test)] + fn with_registration_limit(registration_limit: usize) -> Self { + Self { + registration_limit, + ..Default::default() + } + } + + #[cfg(test)] + pub(crate) fn with_registration_snapshot_hook(hook: MetricRegistrationSnapshotHook) -> Self { + Self { + registration_snapshot_hook: Some(hook), + ..Default::default() + } + } + + pub(crate) fn get_existing_metrics_logs( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + ) -> Option>>> { + self.clients.get_existing_client_with( + TelemetryCachedClientOwner::Runtime(instance_id.clone()), + service, + env, + ) + } + + pub(crate) fn get_or_create_metrics_logs( + &self, + service: &str, + env: &str, + instance_id: &InstanceId, + runtime_meta: &RuntimeMetadata, + get_config: F, + process_tags: Vec, + ) -> Arc>> + where + F: FnOnce() -> Config, + { + self.clients.get_or_create_with( + TelemetryCachedClientOwner::Runtime(instance_id.clone()), + service, + env, + || { + let registrations = self.registered_metrics(instance_id, service, env); + #[cfg(test)] + if let Some(hook) = &self.registration_snapshot_hook { + hook.wait(); + } + let mut client = TelemetryCachedClient::new_metrics_logs( + service, + env, + instance_id, + runtime_meta, + get_config, + process_tags, + ); + for metric in registrations { + client.register_metric(metric); + } + client + }, + ) + } + + #[cfg(test)] + fn remove_metrics_logs_client( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + expected: &Arc>>, + ) { + self.clients.remove_client_with( + TelemetryCachedClientOwner::Runtime(instance_id.clone()), + service, + env, + expected, + ); + } + + pub(crate) fn registered_metrics( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + ) -> Vec { + let scope = TelemetryMetricRegistrationScope::new(instance_id, service, env); + self.registrations + .lock_or_panic() + .get(&scope) + .into_iter() + .flat_map(|metrics| metrics.values().cloned()) + .collect() + } + + #[cfg(test)] + fn registered_metric_names( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + ) -> HashSet { + self.registered_metrics(instance_id, service, env) + .into_iter() + .map(|metric| metric.name) + .collect() + } + + #[cfg(test)] + pub(crate) fn register_metric( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + metric: MetricContext, + ) -> bool { + !matches!( + self.register_metric_with_outcome(instance_id, service, env, metric), + MetricRegistrationOutcome::RejectedCapacity { .. } + ) + } + + fn register_metric_with_outcome( + &self, + instance_id: &InstanceId, + service: &str, + env: &str, + metric: MetricContext, + ) -> MetricRegistrationOutcome { + let scope = TelemetryMetricRegistrationScope::new(instance_id, service, env); + let mut registrations = self.registrations.lock_or_panic(); + let metrics = registrations.entry(scope).or_default(); + let outcome = match metrics.get(&metric.name) { + Some(registered_metric) if metric_contexts_match(registered_metric, &metric) => { + MetricRegistrationOutcome::Unchanged + } + Some(_) => MetricRegistrationOutcome::Changed, + None if metrics.len() >= self.registration_limit => { + MetricRegistrationOutcome::RejectedCapacity { + limit: self.registration_limit, + } + } + None => MetricRegistrationOutcome::Inserted, + }; + if matches!( + outcome, + MetricRegistrationOutcome::Unchanged + | MetricRegistrationOutcome::RejectedCapacity { .. } + ) { + return outcome; + } + metrics.insert(metric.name.clone(), metric.clone()); + drop(registrations); + + if outcome == MetricRegistrationOutcome::Changed { + self.clients + .remove_clients_matching(|owner, client_service, client_env| { + matches!( + owner, + TelemetryCachedClientOwner::Runtime(owner_instance) + if owner_instance.session_id == instance_id.session_id + && client_service == service + && client_env == env + ) + }); + return outcome; + } + + let clients = self + .clients + .inner + .lock_or_panic() + .iter() + .filter_map(|((owner, client_service, client_env), entry)| { + let TelemetryCachedClientOwner::Runtime(owner_instance) = owner else { + return None; + }; + (owner_instance.session_id == instance_id.session_id + && client_service == service + && client_env == env) + .then(|| entry.client.clone()) + }) + .collect::>(); + for client in clients { + if let Some(client) = client.lock_or_panic().as_mut() { + if !client.is_stopping() { + client.register_metric(metric.clone()); + } + } + } + outcome + } +} pub fn path_for_telemetry(service: &str, env: &str) -> CString { let mut hasher = ZwoHasher::default(); service.hash(&mut hasher); env.hash(&mut hasher); - telemetry_path_from_hash(hasher.finish()) -} + let hash = hasher.finish(); -fn telemetry_path_from_hash(hash: u64) -> CString { let mut path = format!( "/ddtl{}-{}", primary_sidecar_identifier(), @@ -861,19 +2237,26 @@ fn telemetry_path_from_hash(hash: u64) -> CString { CString::new(path).unwrap() } -pub fn get_telemetry_action_sender() -> Result> { +pub fn get_telemetry_action_sender() -> Result { TELEMETRY_ACTION_SENDER .get() .cloned() .ok_or_else(|| anyhow!("Telemetry action sender not initialized")) } -pub(crate) fn init_telemetry_sender() -> Option> { +pub(crate) fn init_telemetry_sender( + sidecar: &SidecarServer, +) -> Option> { let (tx, rx) = mpsc::channel(1000); - if TELEMETRY_ACTION_SENDER.set(tx).is_err() { + let sender = TelemetryActionSender { + sender: tx, + lifecycles: sidecar.direct_telemetry_lifecycles.clone(), + }; + if TELEMETRY_ACTION_SENDER.set(sender.clone()).is_err() { warn!("Telemetry action sender already initialized"); return None; } + sidecar.install_direct_telemetry_sender(sender); Some(rx) } @@ -882,15 +2265,17 @@ fn get_telemetry_client( instance_id: &InstanceId, service_name: &str, env_name: &str, -) -> Option>> { - if let Some(existing) = sidecar - .metrics_logs_clients - .get_existing_client(service_name, env_name) +) -> Option>>> { + if let Some(existing) = + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(instance_id, service_name, env_name) { return Some(existing); } - let session = sidecar.get_session(&instance_id.session_id); + let session = sidecar.find_session(&instance_id.session_id)?; + sidecar.find_runtime(instance_id)?; let trace_config = session.get_trace_config(); let runtime_meta = RuntimeMetadata::new( trace_config.language.as_str(), @@ -906,7 +2291,7 @@ fn get_telemetry_client( let process_tags = session.process_tags_with_svc_source(); - Some(sidecar.metrics_logs_clients.get_or_create( + Some(sidecar.metrics_logs_clients.get_or_create_metrics_logs( service_name, env_name, instance_id, @@ -915,3 +2300,2247 @@ fn get_telemetry_client( process_tags, )) } + +#[cfg(test)] +mod tests { + use super::*; + use datadog_ipc::one_way_shared_memory::{open_named_shm, OneWayShmReader}; + use httpmock::{Method::POST, MockServer}; + use libdd_telemetry::data::{Configuration, ConfigurationOrigin, Log, LogLevel}; + use libdd_telemetry::worker::{LifecycleAction, LogIdentifier}; + use std::io::Write; + use std::sync::atomic::AtomicUsize; + use tokio::sync::Barrier; + use tokio::time::{sleep, timeout}; + + const TELEMETRY_PATH: &str = "/telemetry/proxy/api/v2/apmtelemetry"; + + fn test_config(server: &MockServer) -> Config { + let mut config = Config::default(); + config + .set_endpoint_uri(server.url("/").parse().unwrap()) + .unwrap(); + config + } + + fn initial_configuration(name: &str) -> Configuration { + Configuration { + name: name.to_string(), + value: "present".to_string(), + origin: ConfigurationOrigin::Default, + config_id: None, + seq_id: None, + } + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn application_shm_writer_retries_and_publishes_current_state() { + const SERVICE: &str = "shm-retry"; + const ENV: &str = "test"; + + let retry_at = Instant::now(); + let attempts = Arc::new(AtomicUsize::new(0)); + let attempts_for_factory = attempts.clone(); + let path = path_for_telemetry(SERVICE, ENV); + let mut client = TelemetryCachedClient::new_with_shm_factory( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "runtime"), + &RuntimeMetadata::new("php", "8.3", "test"), + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + retry_at, + move |_| { + attempts_for_factory.fetch_add(1, Ordering::Relaxed); + Err(std::io::Error::other("injected failure")) + }, + ) + .unwrap(); + client.shared.config_sent = true; + + client.write_shm_file_at(retry_at + Duration::from_secs(59), |_| { + panic!("retry happened before the deadline") + }); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + + client.write_shm_file_at(retry_at + Duration::from_secs(60), |path| { + OneWayShmWriter::::new(path.clone()) + }); + assert_eq!(attempts.load(Ordering::Relaxed), 1); + + let mut reader = OneWayShmReader::new(open_named_shm(&path).unwrap(), ()); + let shared: TelemetryCachedClientShmData = bincode::deserialize(reader.read().1).unwrap(); + assert!(shared.config_sent); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn pending_application_actions_are_bounded_without_dropping_terminal_promotion() { + const SERVICE: &str = "bounded-pending"; + const ENV: &str = "test"; + const LIMIT: usize = 2; + + let clients = TelemetryCachedClientSet::with_pending_action_limit(LIMIT); + let instance = InstanceId::new("session", "runtime"); + let runtime = RuntimeMetadata::new("php", "8.3", "test"); + let dependency = |name: &str| { + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: name.to_string(), + version: None, + })) + }; + + for name in ["one", "two", "overflow"] { + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime, Vec::new()), + PendingApplicationAction::from_actions( + &instance, + vec![dependency(name)], + &HashMap::new(), + ), + Config::default, + |_, _| panic!("a non-terminal batch must remain pending"), + ); + assert!(matches!(dispatch, ApplicationTelemetryDispatch::Pending)); + } + + let pending = clients.pending.lock_or_panic(); + let stored = &pending + .get(&(SERVICE.to_string(), ENV.to_string())) + .expect("bounded pending lifecycle") + .actions; + assert_eq!(stored.len(), LIMIT); + assert_eq!( + stored + .iter() + .map(|action| match &action.action { + SidecarAction::Telemetry(TelemetryActions::AddDependency(dependency)) => + dependency.name.as_str(), + _ => panic!("only dependencies should be buffered"), + }) + .collect::>(), + ["one", "two"], + "overflow rejects the whole non-terminal batch without disturbing prior startup data" + ); + drop(pending); + + let promoted = Arc::new(Mutex::new(Vec::new())); + let promoted_for_initializer = promoted.clone(); + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime, Vec::new()), + PendingApplicationAction::from_actions( + &instance, + vec![SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop, + ))], + &HashMap::new(), + ), + Config::default, + move |_, actions| { + promoted_for_initializer.lock_or_panic().extend( + actions + .into_iter() + .map(|pending_action| pending_action.action), + ); + true + }, + ); + assert!(matches!( + dispatch, + ApplicationTelemetryDispatch::Ready { created: true, .. } + )); + let promoted = promoted.lock_or_panic(); + assert_eq!(promoted.len(), LIMIT + 1); + assert!(matches!( + promoted.last(), + Some(SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop + ))) + )); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn pending_application_startup_data_uses_global_application_ownership() { + const SERVICE: &str = "shared-pending"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let instance_a = InstanceId::new("session-a", "runtime-a"); + let instance_b = InstanceId::new("session-b", "runtime-b"); + let runtime = RuntimeMetadata::new("php", "8.3", "test"); + let dependency = |name: &str| { + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: name.to_string(), + version: None, + })) + }; + + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance_a, &runtime, Vec::new()), + PendingApplicationAction::from_actions( + &instance_a, + vec![dependency("from-a")], + &HashMap::new(), + ), + Config::default, + |_, _| panic!("dependency-only startup data must remain pending"), + ); + assert!(matches!(dispatch, ApplicationTelemetryDispatch::Pending)); + + let promoted = Arc::new(Mutex::new(Vec::new())); + let promoted_for_initializer = promoted.clone(); + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance_b, &runtime, Vec::new()), + PendingApplicationAction::from_actions( + &instance_b, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + initial_configuration("session-b-config"), + ))], + &HashMap::new(), + ), + Config::default, + move |_, actions| { + promoted_for_initializer + .lock_or_panic() + .extend(actions.into_iter().map(|pending| pending.action)); + false + }, + ); + assert!(matches!( + dispatch, + ApplicationTelemetryDispatch::Ready { created: true, .. } + )); + + let promoted = promoted.lock_or_panic(); + assert!(promoted.iter().any(|action| matches!( + action, + SidecarAction::Telemetry(TelemetryActions::AddDependency(dependency)) + if dependency.name == "from-a" + ))); + assert!(promoted.iter().any(|action| matches!( + action, + SidecarAction::Telemetry(TelemetryActions::AddConfig(configuration)) + if configuration.name == "session-b-config" + ))); + drop(promoted); + + assert!( + clients.pending.lock_or_panic().is_empty(), + "promotion by another session must drain the global application's pending data" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn session_cleanup_waits_for_failed_global_promotion_restore() { + const SERVICE: &str = "cleanup-during-promotion"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let instance_a = InstanceId::new("session-a", "runtime-a"); + let instance_b = InstanceId::new("session-b", "runtime-b"); + let runtime = RuntimeMetadata::new("php", "8.3", "test"); + let dependency = + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: "from-a".to_string(), + version: None, + })); + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance_a, &runtime, Vec::new()), + PendingApplicationAction::from_actions(&instance_a, vec![dependency], &HashMap::new()), + Config::default, + |_, _| panic!("dependency-only startup data must remain pending"), + ); + assert!(matches!(dispatch, ApplicationTelemetryDispatch::Pending)); + + let constructor_entered = Arc::new(std::sync::Barrier::new(2)); + let release_constructor = Arc::new(std::sync::Barrier::new(2)); + let creating_clients = clients.clone(); + let entered_by_constructor = constructor_entered.clone(); + let release_in_constructor = release_constructor.clone(); + let creation = tokio::task::spawn_blocking(move || { + let dispatch = creating_clients.get_or_create_for_actions_with( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance_b, &runtime, Vec::new()), + PendingApplicationAction::from_actions( + &instance_b, + vec![SidecarAction::Telemetry(TelemetryActions::AddConfig( + initial_configuration("session-b-config"), + ))], + &HashMap::new(), + ), + |_, _| panic!("injected construction failure must not initialize a client"), + |_, _| { + entered_by_constructor.wait(); + release_in_constructor.wait(); + Err(anyhow!("injected construction failure")) + }, + ); + assert!(matches!(dispatch, ApplicationTelemetryDispatch::Pending)); + }); + constructor_entered.wait(); + + let cleanup_started = Arc::new(std::sync::Barrier::new(2)); + let cleanup_clients = clients.clone(); + let cleanup_started_in_task = cleanup_started.clone(); + let cleanup = tokio::task::spawn_blocking(move || { + cleanup_started_in_task.wait(); + cleanup_clients.remove_pending_session("session-a"); + }); + cleanup_started.wait(); + tokio::time::sleep(Duration::from_millis(25)).await; + assert!( + !cleanup.is_finished(), + "session cleanup must wait until failed construction restores pending actions" + ); + + release_constructor.wait(); + creation.await.expect("creation task"); + cleanup.await.expect("cleanup task"); + + let pending = clients.pending.lock_or_panic(); + let restored = &pending + .get(&(SERVICE.to_string(), ENV.to_string())) + .expect("session B promotion should remain pending") + .actions; + assert!(restored + .iter() + .all(|action| action.origin.session_id == "session-b")); + assert!(restored + .iter() + .any(PendingApplicationAction::promotes_lifecycle)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn construction_failure_restores_terminal_suffix_for_later_promotion() { + const SERVICE: &str = "failed-construction-suffix"; + const ENV: &str = "test"; + + fn action_names(actions: &[PendingApplicationAction]) -> Vec { + actions + .iter() + .map(|pending_action| match &pending_action.action { + SidecarAction::Telemetry(TelemetryActions::AddDependency(dependency)) => { + dependency.name.clone() + } + SidecarAction::Telemetry(TelemetryActions::Lifecycle( + LifecycleAction::Stop, + )) => "stop".to_string(), + action => panic!("unexpected action in constructor failure test: {action:?}"), + }) + .collect() + } + + let clients = TelemetryCachedClientSet::with_pending_action_limit(2); + let instance = InstanceId::new("session", "runtime"); + let runtime = RuntimeMetadata::new("php", "8.3", "test"); + let actions = PendingApplicationAction::from_actions( + &instance, + vec![ + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: "before-stop".to_string(), + version: None, + })), + SidecarAction::Telemetry(TelemetryActions::Lifecycle(LifecycleAction::Stop)), + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: "after-stop".to_string(), + version: None, + })), + SidecarAction::Telemetry(TelemetryActions::AddDependency(data::Dependency { + name: "overflow".to_string(), + version: None, + })), + ], + &HashMap::new(), + ); + + let failed_dispatch = clients.get_or_create_for_actions_with( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime, Vec::new()), + actions, + |_, _| panic!("injected construction failure must not initialize a client"), + |_, _| { + Err(anyhow!( + "injected application telemetry client creation failure" + )) + }, + ); + assert!(matches!( + failed_dispatch, + ApplicationTelemetryDispatch::Pending + )); + { + let pending = clients.pending.lock_or_panic(); + let restored = &pending + .get(&(SERVICE.to_string(), ENV.to_string())) + .expect("failed construction should restore the pending lifecycle") + .actions; + assert_eq!( + action_names(restored), + vec![ + "before-stop".to_string(), + "stop".to_string(), + "after-stop".to_string(), + ], + "construction failure must restore a bounded source-ordered lifecycle" + ); + } + + let initialized = Arc::new(Mutex::new(Vec::new())); + let initialized_for_callback = initialized.clone(); + let dispatch = clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new(SERVICE, ENV, &instance, &runtime, Vec::new()), + Vec::new(), + Config::default, + move |_, actions| { + *initialized_for_callback.lock_or_panic() = action_names(&actions); + false + }, + ); + let ApplicationTelemetryDispatch::Ready { + actions, created, .. + } = dispatch + else { + panic!("restored lifecycle should promote after construction recovers"); + }; + assert!(created); + assert_eq!( + *initialized.lock_or_panic(), + vec!["before-stop".to_string(), "stop".to_string()] + ); + assert_eq!(action_names(&actions), vec!["after-stop".to_string()]); + } + + fn internal_log(message: &str) -> InternalTelemetryAction { + InternalTelemetryAction::TelemetryAction(TelemetryActions::AddLog(( + LogIdentifier { identifier: 1 }, + Log { + message: message.to_string(), + level: LogLevel::Debug, + count: 1, + stack_trace: None, + tags: String::new(), + is_sensitive: false, + is_crash: false, + }, + ))) + } + + fn metric(name: &str) -> MetricContext { + MetricContext { + name: name.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + } + } + + #[derive(Clone)] + struct CapturedLogWriter(Arc>>); + + impl Write for CapturedLogWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock_or_panic().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[tokio::test] + async fn metric_capacity_rejection_is_not_logged_as_registration_success() { + let captured = Arc::new(Mutex::new(Vec::new())); + let writer_capture = captured.clone(); + let subscriber = tracing_subscriber::fmt() + .without_time() + .with_ansi(false) + .with_max_level(tracing::Level::DEBUG) + .with_writer(move || CapturedLogWriter(writer_capture.clone())) + .finish(); + let _subscriber_guard = tracing::subscriber::set_default(subscriber); + let mut sidecar = SidecarServer::default(); + sidecar.metrics_logs_clients = MetricsLogsClientSet::with_registration_limit(1); + let instance = InstanceId::new("capacity-session", "capacity-runtime"); + let mut active_client = BatchDirectTelemetryClient::Unavailable; + + deliver_batch( + vec![ + InternalTelemetryAction::RegisterTelemetryMetric(metric("accepted.metric")), + InternalTelemetryAction::RegisterTelemetryMetric(metric("rejected.metric")), + ], + &sidecar, + &instance, + "capacity-service", + "capacity-env", + &mut active_client, + ) + .await; + + let output = String::from_utf8(captured.lock_or_panic().clone()).expect("UTF-8 logs"); + assert!(output.contains( + "Registered telemetry metric: session=capacity-session \ + service=capacity-service env=capacity-env metric=accepted.metric" + )); + assert!(output.contains( + "Rejected telemetry metric registration: session=capacity-session \ + service=capacity-service env=capacity-env metric=rejected.metric capacity=1" + )); + assert!(!output.contains( + "Registered telemetry metric: session=capacity-session \ + service=capacity-service env=capacity-env metric=rejected.metric" + )); + } + + async fn metric_context_count(worker: &TelemetryWorkerHandle) -> u32 { + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .expect("metric worker should collect stats"); + rx.await + .expect("metric worker should return stats") + .metric_contexts + } + + fn metric_context_key( + client: &Arc>>, + metric_name: &str, + ) -> ContextKey { + let action = client + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .to_telemetry_point((metric_name.to_string(), 1.0, Vec::new())) + .expect("registered metric should produce a point"); + let TelemetryActions::AddPoint((_, context_key, _)) = action else { + panic!("metric point should use an AddPoint action"); + }; + context_key + } + + #[tokio::test] + async fn deferred_batches_are_scoped_by_instance() { + let instance_a = InstanceId::new("session", "runtime-a"); + let instance_b = InstanceId::new("session", "runtime-b"); + let mut pending = vec![PerClientTelemetryBatch { + key: ( + instance_a.clone(), + "shared-service".to_string(), + "test".to_string(), + None, + ), + actions: VecDeque::from([DirectTelemetryActions { + actions: InternalTelemetryActions { + instance_id: instance_a, + service_name: "shared-service".to_string(), + env_name: "test".to_string(), + actions: vec![internal_log("owner-a")], + }, + generation: None, + }]), + attempts_left: 2, + next_attempt_at: TokioInstant::now() + Duration::from_secs(60), + }]; + let (tx, mut rx) = mpsc::channel(1); + tx.send(DirectTelemetryMessage::Actions(DirectTelemetryActions { + actions: InternalTelemetryActions { + instance_id: instance_b.clone(), + service_name: "shared-service".to_string(), + env_name: "test".to_string(), + actions: vec![internal_log("owner-b")], + }, + generation: None, + })) + .await + .unwrap(); + + let batch = next_entry(&mut pending, &mut rx) + .await + .expect("second owner should remain a fresh batch"); + let ReceiverEntry::Batch(TelemetryBatch::Fresh(batch)) = batch else { + panic!("different owners must not share a deferred batch"); + }; + assert_eq!(batch.actions.instance_id, instance_b); + assert_eq!(pending[0].actions.len(), 1); + } + + #[test] + fn full_direct_channel_returns_the_unsent_batch() { + let (sender, _receiver) = mpsc::channel(1); + let sender = TelemetryActionSender { + sender, + lifecycles: DirectTelemetryLifecycleRegistry::default(), + }; + sender + .try_send(InternalTelemetryActions { + instance_id: InstanceId::new("session", "first"), + service_name: "service".to_string(), + env_name: "test".to_string(), + actions: Vec::new(), + }) + .expect("first batch should fill the channel"); + + let error = sender + .try_send(InternalTelemetryActions { + instance_id: InstanceId::new("session", "retry"), + service_name: "retry-service".to_string(), + env_name: "retry-env".to_string(), + actions: vec![internal_log("retry me")], + }) + .expect_err("second batch should be returned when the channel is full"); + let mpsc::error::TrySendError::Full(recovered) = error else { + panic!("the open channel should report full"); + }; + assert_eq!(recovered.instance_id.runtime_id, "retry"); + assert_eq!(recovered.service_name, "retry-service"); + assert_eq!(recovered.env_name, "retry-env"); + assert_eq!(recovered.actions.len(), 1); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn direct_batch_before_first_runtime_creation_is_retried() { + const SERVICE: &str = "initial-direct-batch"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("arrives before first runtime")], + }) + .await + .expect("initial direct action"); + sender.barrier().await.expect("first delivery attempt"); + + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + sleep(TelemetryBatch::RETRY_DELAY + Duration::from_millis(50)).await; + sender.barrier().await.expect("retry deadline"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_some(), + "a batch submitted before initial runtime creation should be retried" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn runtime_retirement_preserves_unrelated_pre_runtime_batch() { + const SERVICE: &str = "unrelated-pre-runtime-batch"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let retiring_instance = InstanceId::new("retiring-session", "runtime"); + let unrelated_instance = InstanceId::new("unrelated-session", "runtime"); + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + + sender + .send_actions(InternalTelemetryActions { + instance_id: unrelated_instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("unrelated deferred batch")], + }) + .await + .expect("unrelated direct action"); + sender.barrier().await.expect("first delivery attempt"); + + let retirement = sidecar + .direct_telemetry_lifecycles + .begin_retire_runtimes(&HashSet::from([retiring_instance])); + sender + .retire(DirectTelemetryRetirement::Runtimes(retirement.clone())) + .await + .expect("retire a different runtime"); + sidecar + .direct_telemetry_lifecycles + .finish_retire_runtimes(&retirement, |_| {}); + + let session = sidecar.get_session(&unrelated_instance.session_id); + sidecar.get_runtime(&unrelated_instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + sleep(TelemetryBatch::RETRY_DELAY + Duration::from_millis(50)).await; + sender.barrier().await.expect("retry deadline"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&unrelated_instance, SERVICE, ENV) + .is_some(), + "retiring one runtime must not purge another runtime's deferred batch" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn runtime_retirement_drops_pre_activation_batch_before_instance_reuse() { + const SERVICE: &str = "retired-pre-activation-batch"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("belongs to original runtime")], + }) + .await + .expect("pre-activation direct action"); + sender.barrier().await.expect("first delivery attempt"); + + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + let retirement = sidecar + .direct_telemetry_lifecycles + .begin_retire_runtimes(&HashSet::from([instance.clone()])); + sender + .retire(DirectTelemetryRetirement::Runtimes(retirement.clone())) + .await + .expect("retire original runtime"); + sidecar + .direct_telemetry_lifecycles + .finish_retire_runtimes(&retirement, |ready| { + assert!(ready.contains(&instance)); + session.take_runtime(&instance.runtime_id); + }); + + sidecar.get_runtime(&instance); + sleep(TelemetryBatch::RETRY_DELAY + Duration::from_millis(50)).await; + sender.barrier().await.expect("retry deadline"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none(), + "a pre-activation batch from the retired runtime must not attach to its replacement" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn retired_direct_batch_is_not_delivered_to_reused_instance_id() { + const SERVICE: &str = "retired-direct-batch"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + + let retirement = sidecar + .direct_telemetry_lifecycles + .begin_retire_runtimes(&HashSet::from([instance.clone()])); + session.take_runtime(&instance.runtime_id); + sender + .retire(DirectTelemetryRetirement::Runtimes(retirement.clone())) + .await + .expect("retire original lifecycle"); + sidecar + .direct_telemetry_lifecycles + .finish_retire_runtimes(&retirement, |_| {}); + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("belongs to retired lifecycle")], + }) + .await + .expect("late direct action"); + sender.barrier().await.expect("late batch delivery attempt"); + + sidecar.get_runtime(&instance); + sleep(TelemetryBatch::RETRY_DELAY + Duration::from_millis(50)).await; + sender.barrier().await.expect("retry deadline"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none(), + "a batch from the retired generation must not attach to a reused instance id" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn retired_session_generation_is_not_reused() { + const SERVICE: &str = "retired-session-batch"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + let retired_generation = sidecar + .direct_telemetry_lifecycles + .generation(&instance) + .expect("initial runtime generation"); + + sidecar + .direct_telemetry_lifecycles + .retire_session(&instance.session_id); + sidecar.get_runtime(&instance); + + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + sender + .sender + .send(DirectTelemetryMessage::Actions(DirectTelemetryActions { + actions: InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("belongs to retired session")], + }, + generation: Some(retired_generation), + })) + .await + .expect("delayed old-session batch"); + sender + .barrier() + .await + .expect("old-session delivery attempt"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none(), + "a reused session/runtime id must not reactivate an old generation" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn direct_batch_retries_while_existing_runtime_waits_for_session_config() { + const SERVICE: &str = "direct-before-config"; + const ENV: &str = "test"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + let (sender, receiver) = direct_telemetry_channel(&sidecar); + let receiver_task = tokio::spawn(telemetry_action_receiver_task(sidecar.clone(), receiver)); + + sender + .send_actions(InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log("waits for configuration")], + }) + .await + .expect("pre-config direct action"); + sender.barrier().await.expect("first delivery attempt"); + assert!(sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none()); + + *session.session_config.lock_or_panic() = Some(Config::default()); + sleep(TelemetryBatch::RETRY_DELAY + Duration::from_millis(50)).await; + sender.barrier().await.expect("retry delivery"); + + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_some(), + "an existing lifecycle should deliver once its session config becomes available" + ); + receiver_task.abort(); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metric_registration_only_batch_does_not_create_runtime_worker() { + const SERVICE: &str = "registration-only"; + const ENV: &str = "test"; + const METRIC: &str = "registration.only"; + + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("session", "runtime"); + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(Config::default()); + + let delivery = TelemetryBatch::Fresh(DirectTelemetryActions { + generation: sidecar.direct_telemetry_lifecycles.generation(&instance), + actions: InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![InternalTelemetryAction::RegisterTelemetryMetric(metric( + METRIC, + ))], + }, + }) + .deliver(&sidecar) + .await; + + assert!(delivery.is_ok()); + assert_eq!( + sidecar + .metrics_logs_clients + .registered_metric_names(&instance, SERVICE, ENV), + HashSet::from([METRIC.to_string()]) + ); + assert!( + sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .is_none(), + "registration should update the session definition map without spawning a worker" + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn internal_log_after_app_stop_uses_metrics_logs_worker() { + const SERVICE: &str = "internal-before-config"; + const ENV: &str = "test"; + const LOG_MESSAGE: &str = "queued before configuration"; + + let http_server = MockServer::start_async().await; + let app_started_with_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_includes("\"name\":\"initial_config\""); + then.status(202); + }) + .await; + let app_started_without_config = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\"") + .body_excludes("\"name\":\"initial_config\""); + then.status(202); + }) + .await; + let log_request = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes(LOG_MESSAGE); + then.status(202); + }) + .await; + let app_closing = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-closing\""); + then.status(202); + }) + .await; + + let sidecar = SidecarServer::default(); + let instance_id = InstanceId::new("session", "runtime"); + let session = sidecar.get_session(&instance_id.session_id); + sidecar.get_runtime(&instance_id); + *session.session_config.lock_or_panic() = Some(test_config(&http_server)); + let app_client = sidecar + .telemetry_clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &instance_id, + &RuntimeMetadata::new("php", "8.3", "test"), + Vec::new(), + ), + || test_config(&http_server), + InitialTelemetryData { + configurations: vec![initial_configuration("initial_config")], + ..Default::default() + }, + ) + .expect("application telemetry worker"); + let app_worker = { + let client = app_client.lock_or_panic(); + client + .as_ref() + .expect("app telemetry client") + .worker + .clone() + }; + app_worker.send_stop().unwrap(); + sidecar + .telemetry_clients + .remove_telemetry_client(SERVICE, ENV, &app_client); + + let batch = TelemetryBatch::Fresh(DirectTelemetryActions { + generation: sidecar.direct_telemetry_lifecycles.generation(&instance_id), + actions: InternalTelemetryActions { + instance_id: instance_id.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log(LOG_MESSAGE)], + }, + }); + + let metrics_logs_client = get_telemetry_client(&sidecar, &instance_id, SERVICE, ENV) + .expect("session config should allow a metrics/logs client"); + assert!(!Arc::ptr_eq(&app_client, &metrics_logs_client)); + let worker = metrics_logs_client + .lock_or_panic() + .as_ref() + .expect("metrics/logs telemetry client") + .worker + .clone(); + + assert!(batch.deliver(&sidecar).await.is_ok()); + worker + .send_msg(TelemetryActions::Lifecycle( + LifecycleAction::FlushMetricAggr, + )) + .await + .unwrap(); + worker + .send_msg(TelemetryActions::Lifecycle(LifecycleAction::FlushData)) + .await + .unwrap(); + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .unwrap(); + rx.await.unwrap(); + + timeout(Duration::from_secs(5), async { + while app_started_with_config.calls_async().await != 1 + || log_request.calls_async().await != 1 + || app_closing.calls_async().await != 1 + { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("app lifecycle and late internal log should arrive"); + + assert_eq!(app_started_without_config.calls_async().await, 0); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + #[cfg_attr(miri, ignore)] + async fn concurrent_same_key_creates_one_worker() { + const CALLERS: usize = 32; + const SERVICE: &str = "concurrent-client-creation"; + const ENV: &str = "test"; + + let http_server = MockServer::start_async().await; + let app_started = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"request_type\":\"app-started\""); + then.status(202); + }) + .await; + let clients = TelemetryCachedClientSet::default(); + let barrier = Arc::new(Barrier::new(CALLERS)); + let config = test_config(&http_server); + + let tasks = (0..CALLERS).map(|index| { + let clients = clients.clone(); + let barrier = barrier.clone(); + let config = config.clone(); + tokio::spawn(async move { + let instance_id = InstanceId::new("session", &format!("runtime-{index}")); + barrier.wait().await; + clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &instance_id, + &RuntimeMetadata::new("php", "8.3", "test"), + Vec::new(), + ), + || config, + InitialTelemetryData { + configurations: vec![initial_configuration("concurrent_config")], + ..Default::default() + }, + ) + .expect("concurrent application telemetry worker") + }) + }); + let returned_clients = futures::future::join_all(tasks) + .await + .into_iter() + .map(Result::unwrap) + .collect::>(); + + let first = &returned_clients[0]; + assert!( + returned_clients + .iter() + .all(|client| Arc::ptr_eq(first, client)), + "all same-key callers should receive the same telemetry client" + ); + assert_eq!(clients.inner.lock_or_panic().len(), 1); + + timeout(Duration::from_secs(5), async { + while app_started.calls_async().await != 1 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("exactly one app-started request should arrive"); + assert_eq!(app_started.calls_async().await, 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[cfg_attr(miri, ignore)] + async fn promotion_initialization_precedes_visible_duplicate_integration() { + const SERVICE: &str = "atomic-promotion"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let initial_instance = InstanceId::new("session", "initial-runtime"); + let duplicate_instance = InstanceId::new("session", "duplicate-runtime"); + let integration = data::Integration { + name: "initial-integration".to_string(), + enabled: true, + version: None, + compatible: None, + auto_enabled: None, + }; + let barrier = Arc::new(std::sync::Barrier::new(2)); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + + let initial_clients = clients.clone(); + let initial_barrier = barrier.clone(); + let initial_integration = integration.clone(); + let initial = tokio::task::spawn_blocking(move || { + initial_clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &initial_instance, + &RuntimeMetadata::new("php", "8.3", "test"), + Vec::new(), + ), + PendingApplicationAction::from_actions( + &initial_instance, + vec![ + SidecarAction::Telemetry(TelemetryActions::AddIntegration( + initial_integration, + )), + SidecarAction::Telemetry(TelemetryActions::AddConfig( + initial_configuration("initial-config"), + )), + ], + &HashMap::new(), + ), + Config::default, + move |_, _| { + started_tx + .send(()) + .expect("test receiver should be available"); + initial_barrier.wait(); + false + }, + ) + }); + + started_rx + .recv_timeout(Duration::from_secs(1)) + .expect("promotion initializer should begin"); + + let duplicate_clients = clients.clone(); + let duplicate_integration = integration.clone(); + let duplicate = tokio::task::spawn_blocking(move || { + duplicate_clients.get_or_create_for_actions( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &duplicate_instance, + &RuntimeMetadata::new("php", "8.3", "test"), + Vec::new(), + ), + PendingApplicationAction::from_actions( + &duplicate_instance, + vec![SidecarAction::Telemetry(TelemetryActions::AddIntegration( + duplicate_integration, + ))], + &HashMap::new(), + ), + Config::default, + |_, _| panic!("active client should not be initialized again"), + ) + }); + + sleep(Duration::from_millis(50)).await; + assert!( + !duplicate.is_finished(), + "a duplicate integration must wait until startup initialization completes" + ); + barrier.wait(); + + let initial = initial.await.expect("initial promotion task"); + assert!(matches!( + initial, + ApplicationTelemetryDispatch::Ready { created: true, .. } + )); + let ApplicationTelemetryDispatch::Ready { + client, + actions, + created, + .. + } = duplicate.await.expect("duplicate task") + else { + panic!("duplicate integration should find the published client"); + }; + assert!(!created); + assert_eq!(actions.len(), 1); + assert!(client + .lock_or_panic() + .as_ref() + .expect("published telemetry client") + .shared + .integrations + .contains(&integration)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn stopping_client_is_atomically_replaced() { + const SERVICE: &str = "stale-removal"; + const ENV: &str = "test"; + + let clients = TelemetryCachedClientSet::default(); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let old = clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "old"), + &runtime_metadata, + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + ) + .expect("old application telemetry worker"); + + old.lock_or_panic() + .as_mut() + .expect("old telemetry client") + .mark_stopping(); + let replacement = clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "replacement"), + &runtime_metadata, + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + ) + .expect("replacement application telemetry worker"); + assert!(!Arc::ptr_eq(&old, &replacement)); + const REPLACEMENT_STATE: &[u8] = b"replacement state"; + { + let replacement_client = replacement.lock_or_panic(); + let ApplicationShmState::Ready(shm_writer) = &replacement_client + .as_ref() + .expect("replacement telemetry client") + .shm_state + else { + panic!("replacement shared-memory writer should be ready"); + }; + shm_writer.write(REPLACEMENT_STATE); + } + + old.lock_or_panic().take(); + let mut reader = OneWayShmReader::new( + open_named_shm(&path_for_telemetry(SERVICE, ENV)) + .expect("replacement shared-memory name should remain available"), + (), + ); + assert_eq!(reader.read().1, REPLACEMENT_STATE); + + clients.remove_telemetry_client(SERVICE, ENV, &old); + + let cached = clients + .get_existing_client(SERVICE, ENV) + .expect("replacement client should remain cached"); + assert!(Arc::ptr_eq(&replacement, &cached)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn ttl_eviction_retires_application_shm_before_replacement() { + const SERVICE: &str = "ttl-shm-retirement"; + const ENV: &str = "test"; + const REPLACEMENT_STATE: &[u8] = b"replacement survives old owner"; + + let clients = TelemetryCachedClientSet::with_cleanup(Duration::from_secs(60)); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let old = clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "old-runtime"), + &runtime_metadata, + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + ) + .expect("old application telemetry worker"); + let retained_old_owner = old.clone(); + let now = Instant::now(); + clients + .inner + .lock_or_panic() + .values_mut() + .for_each(|entry| entry.last_used = now - Duration::from_secs(61)); + + clients.evict_expired_at(now, Duration::from_secs(60)); + assert!(old + .lock_or_panic() + .as_ref() + .expect("externally retained client") + .is_stopping()); + + let replacement = clients + .get_or_create( + TelemetryWorkerMetadata::new( + SERVICE, + ENV, + &InstanceId::new("session", "replacement-runtime"), + &runtime_metadata, + Vec::new(), + ), + Config::default, + InitialTelemetryData::default(), + ) + .expect("replacement application telemetry worker"); + { + let replacement = replacement.lock_or_panic(); + let ApplicationShmState::Ready(writer) = + &replacement.as_ref().expect("replacement client").shm_state + else { + panic!("replacement shared-memory writer should be ready"); + }; + writer.write(REPLACEMENT_STATE); + } + + drop(retained_old_owner); + old.lock_or_panic().take(); + + let mut reader = OneWayShmReader::new( + open_named_shm(&path_for_telemetry(SERVICE, ENV)) + .expect("replacement shared-memory name should remain available"), + (), + ); + assert_eq!(reader.read().1, REPLACEMENT_STATE); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metrics_logs_cache_replays_registrations_after_eviction() { + const SERVICE: &str = "persistent-metrics"; + const ENV: &str = "test"; + const METRIC: &str = "persistent.metric"; + + let http_server = MockServer::start_async().await; + let clients = MetricsLogsClientSet::default(); + let instance_id = InstanceId::new("session", "runtime"); + + let client = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance_id, + &RuntimeMetadata::new("php", "8.3", "test"), + || test_config(&http_server), + Vec::new(), + ); + clients.register_metric( + &instance_id, + SERVICE, + ENV, + MetricContext { + name: METRIC.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + ); + let stale_last_used = Instant::now(); + sleep(Duration::from_millis(1)).await; + clients + .clients + .inner + .lock_or_panic() + .get_mut(&( + TelemetryCachedClientOwner::Runtime(instance_id.clone()), + SERVICE.to_string(), + ENV.to_string(), + )) + .expect("cached entry") + .last_used = stale_last_used; + + let cached = clients + .get_existing_metrics_logs(&instance_id, SERVICE, ENV) + .expect("persistent cache entry"); + assert!(Arc::ptr_eq(&client, &cached)); + assert!(cached + .lock_or_panic() + .as_ref() + .expect("metrics/logs client") + .telemetry_metrics + .contains_key(METRIC)); + assert!( + clients + .clients + .inner + .lock_or_panic() + .get(&( + TelemetryCachedClientOwner::Runtime(instance_id.clone()), + SERVICE.to_string(), + ENV.to_string(), + )) + .expect("cached entry") + .last_used + > stale_last_used + ); + + clients.remove_metrics_logs_client(&instance_id, SERVICE, ENV, &client); + let replacement = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance_id, + &RuntimeMetadata::new("php", "8.3", "test"), + || test_config(&http_server), + Vec::new(), + ); + assert!(!Arc::ptr_eq(&client, &replacement)); + assert!(replacement + .lock_or_panic() + .as_ref() + .expect("replacement metrics/logs client") + .telemetry_metrics + .contains_key(METRIC)); + assert_eq!( + clients.registered_metric_names(&instance_id, SERVICE, ENV), + HashSet::from([METRIC.to_string()]) + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metric_registration_is_broadcast_to_existing_matching_runtimes() { + const SERVICE: &str = "shared-appsec-service"; + const ENV: &str = "prod"; + const METRIC: &str = "waf.requests"; + + let server = MockServer::start_async().await; + let clients = MetricsLogsClientSet::default(); + let runtime_meta = RuntimeMetadata::new("php", "8.3", "test"); + let runtime_a = InstanceId::new("session", "runtime-a"); + let runtime_b = InstanceId::new("session", "runtime-b"); + let other_session = InstanceId::new("other-session", "runtime-c"); + let other_service = InstanceId::new("session", "runtime-d"); + let other_env = InstanceId::new("session", "runtime-e"); + + let client_a = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &runtime_a, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + let client_b = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &runtime_b, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + let client_other_session = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &other_session, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + let client_other_service = clients.get_or_create_metrics_logs( + "other-service", + ENV, + &other_service, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + let client_other_env = clients.get_or_create_metrics_logs( + SERVICE, + "other-env", + &other_env, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + + assert!(clients.register_metric( + &runtime_a, + SERVICE, + ENV, + MetricContext { + name: METRIC.to_string(), + tags: Vec::new(), + metric_type: libdd_telemetry::data::metrics::MetricType::Count, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Appsec, + }, + )); + + for client in [&client_a, &client_b] { + assert!(client + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .to_telemetry_point((METRIC.to_string(), 1.0, Vec::new())) + .is_some()); + } + for client in [ + &client_other_session, + &client_other_service, + &client_other_env, + ] { + assert!(client + .lock_or_panic() + .as_ref() + .expect("nonmatching runtime worker") + .to_telemetry_point((METRIC.to_string(), 1.0, Vec::new())) + .is_none()); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[cfg_attr(miri, ignore)] + async fn metric_registration_during_worker_creation_is_replayed() { + const SERVICE: &str = "creation-race-service"; + const ENV: &str = "test"; + const METRIC: &str = "creation.race.metric"; + + let hook = MetricRegistrationSnapshotHook::new(); + let clients = MetricsLogsClientSet::with_registration_snapshot_hook(hook.clone()); + let instance_id = InstanceId::new("session", "runtime"); + let creating_clients = clients.clone(); + let creating_instance = instance_id.clone(); + let creation = tokio::task::spawn_blocking(move || { + creating_clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &creating_instance, + &RuntimeMetadata::new("php", "8.3", "test"), + Config::default, + Vec::new(), + ) + }); + + hook.snapshot_taken.wait(); + let registering_clients = clients.clone(); + let registering_instance = instance_id.clone(); + let registration = tokio::task::spawn_blocking(move || { + registering_clients.register_metric(®istering_instance, SERVICE, ENV, metric(METRIC)) + }); + timeout(Duration::from_secs(1), async { + while !clients + .registered_metric_names(&instance_id, SERVICE, ENV) + .contains(METRIC) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("registration should be stored before worker creation resumes"); + + hook.resume_creation.wait(); + let client = creation.await.expect("worker creation task"); + assert!(registration.await.expect("registration task")); + assert!(client + .lock_or_panic() + .as_ref() + .expect("new runtime worker") + .to_telemetry_point((METRIC.to_string(), 1.0, Vec::new())) + .is_some()); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metric_registrations_do_not_cross_sessions() { + const SERVICE: &str = "shared-appsec-service"; + const ENV: &str = "prod"; + const METRIC: &str = "waf.requests"; + + let server = MockServer::start_async().await; + let clients = MetricsLogsClientSet::default(); + let runtime_meta = RuntimeMetadata::new("php", "8.3", "test"); + let runtime_a = InstanceId::new("session-a", "runtime-a"); + let runtime_b = InstanceId::new("session-b", "runtime-b"); + + let client_a = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &runtime_a, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + assert!(clients.register_metric(&runtime_a, SERVICE, ENV, metric(METRIC))); + assert!(client_a + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .telemetry_metrics + .contains_key(METRIC)); + + let client_b = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &runtime_b, + &runtime_meta, + || test_config(&server), + Vec::new(), + ); + assert!(!client_b + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .telemetry_metrics + .contains_key(METRIC)); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn runtime_and_session_cleanup_remove_owned_state() { + const SERVICE: &str = "cleanup-service"; + const ENV: &str = "test"; + + let clients = MetricsLogsClientSet::default(); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let session_a_runtime_a = InstanceId::new("session-a", "runtime-a"); + let session_a_runtime_b = InstanceId::new("session-a", "runtime-b"); + let session_b_runtime = InstanceId::new("session-b", "runtime"); + + for instance_id in [ + &session_a_runtime_a, + &session_a_runtime_b, + &session_b_runtime, + ] { + clients.get_or_create_metrics_logs( + SERVICE, + ENV, + instance_id, + &runtime_metadata, + Config::default, + Vec::new(), + ); + } + assert!(clients.register_metric( + &session_a_runtime_a, + SERVICE, + ENV, + metric("cleanup.metric"), + )); + assert!(clients.register_metric( + &session_b_runtime, + SERVICE, + ENV, + metric("cleanup.metric"), + )); + + clients.remove_runtime(&session_a_runtime_a); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_a, SERVICE, ENV) + .is_none()); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_b, SERVICE, ENV) + .is_some()); + clients.remove_runtime(&session_a_runtime_a); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_a, SERVICE, ENV) + .is_none()); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_b, SERVICE, ENV) + .is_some()); + + clients.remove_session("session-a"); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_b, SERVICE, ENV) + .is_none()); + assert!(clients + .get_existing_metrics_logs(&session_b_runtime, SERVICE, ENV) + .is_some()); + assert!(clients + .registered_metrics(&session_a_runtime_b, SERVICE, ENV) + .is_empty()); + clients.remove_session("session-a"); + assert!(clients + .get_existing_metrics_logs(&session_a_runtime_b, SERVICE, ENV) + .is_none()); + assert!(clients + .registered_metrics(&session_a_runtime_b, SERVICE, ENV) + .is_empty()); + assert!(clients + .get_existing_metrics_logs(&session_b_runtime, SERVICE, ENV) + .is_some()); + assert!(!clients + .registered_metrics(&session_b_runtime, SERVICE, ENV) + .is_empty()); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn identical_metric_registration_keeps_the_existing_worker_context() { + const SERVICE: &str = "identical-metric-service"; + const ENV: &str = "test"; + const METRIC: &str = "identical.metric"; + + let server = MockServer::start_async().await; + let clients = MetricsLogsClientSet::default(); + let instance = InstanceId::new("session", "runtime"); + let client = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance, + &RuntimeMetadata::new("php", "8.3", "test"), + || test_config(&server), + Vec::new(), + ); + let worker = client + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .worker + .clone(); + let initial_context_count = metric_context_count(&worker).await; + + assert!(clients.register_metric(&instance, SERVICE, ENV, metric(METRIC))); + let first_context = metric_context_key(&client, METRIC); + let registered_context_count = metric_context_count(&worker).await; + assert_eq!(registered_context_count, initial_context_count + 1); + + assert!(clients.register_metric(&instance, SERVICE, ENV, metric(METRIC))); + assert_eq!(metric_context_key(&client, METRIC), first_context); + assert_eq!( + metric_context_count(&worker).await, + registered_context_count + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn direct_batch_reuses_client_lookup_and_refreshes_after_changed_definition() { + const SERVICE: &str = "batch-local-worker"; + const ENV: &str = "test"; + const METRIC: &str = "batch.local.metric"; + + let http_server = MockServer::start_async().await; + let gauge_metric = http_server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes(format!("\"metric\":\"{METRIC}\"")) + .body_includes("\"type\":\"gauge\""); + then.status(202); + }) + .await; + let sidecar = SidecarServer::default(); + let instance = InstanceId::new("batch-local-session", "batch-local-runtime"); + let session = sidecar.get_session(&instance.session_id); + sidecar.get_runtime(&instance); + *session.session_config.lock_or_panic() = Some(test_config(&http_server)); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + + assert!(sidecar.metrics_logs_clients.register_metric( + &instance, + SERVICE, + ENV, + metric(METRIC) + )); + let old_client = sidecar.metrics_logs_clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance, + &runtime_metadata, + || test_config(&http_server), + Vec::new(), + ); + sidecar + .metrics_logs_clients + .clients + .cache_lookup_count + .store(0, Ordering::Relaxed); + + let mut changed_metric = metric(METRIC); + changed_metric.metric_type = libdd_telemetry::data::metrics::MetricType::Gauge; + let delivery = TelemetryBatch::Fresh(DirectTelemetryActions { + generation: sidecar.direct_telemetry_lifecycles.generation(&instance), + actions: InternalTelemetryActions { + instance_id: instance.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![ + InternalTelemetryAction::AddMetricPoint((1.0, METRIC.to_string(), Vec::new())), + InternalTelemetryAction::AddMetricPoint((2.0, METRIC.to_string(), Vec::new())), + InternalTelemetryAction::RegisterTelemetryMetric(changed_metric), + InternalTelemetryAction::AddMetricPoint((3.0, METRIC.to_string(), Vec::new())), + InternalTelemetryAction::AddMetricPoint((4.0, METRIC.to_string(), Vec::new())), + InternalTelemetryAction::TelemetryAction(TelemetryActions::Lifecycle( + LifecycleAction::FlushMetricAggr, + )), + InternalTelemetryAction::TelemetryAction(TelemetryActions::Lifecycle( + LifecycleAction::FlushData, + )), + ], + }, + }) + .deliver(&sidecar) + .await; + assert!(delivery.is_ok()); + + assert_eq!( + sidecar + .metrics_logs_clients + .clients + .cache_lookup_count + .load(Ordering::Relaxed), + 2, + "one initial lookup and one post-definition-change refresh are sufficient" + ); + assert!( + old_client.lock_or_panic().is_none(), + "the changed definition should retire the original worker" + ); + let replacement = sidecar + .metrics_logs_clients + .get_existing_metrics_logs(&instance, SERVICE, ENV) + .expect("same-batch points should create a replacement worker"); + assert!(!Arc::ptr_eq(&old_client, &replacement)); + let worker = replacement + .lock_or_panic() + .as_ref() + .expect("replacement worker") + .worker + .clone(); + let (sender, receiver) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(sender)) + .await + .expect("replacement worker should collect stats"); + receiver + .await + .expect("replacement worker should return stats"); + timeout(Duration::from_secs(5), async { + while gauge_metric.calls_async().await != 1 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("post-change points should use the replacement gauge context"); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn changed_metric_definitions_roll_workers_without_context_growth() { + const SERVICE: &str = "changed-metric-service"; + const ENV: &str = "test"; + const METRIC: &str = "changing.metric"; + + let clients = MetricsLogsClientSet::default(); + let instance = InstanceId::new("session", "runtime"); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let mut current = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance, + &runtime_metadata, + Config::default, + Vec::new(), + ); + + for revision in 0..20 { + let previous = current.clone(); + let mut definition = metric(METRIC); + definition.metric_type = if revision % 2 == 0 { + libdd_telemetry::data::metrics::MetricType::Count + } else { + libdd_telemetry::data::metrics::MetricType::Gauge + }; + assert!(clients.register_metric(&instance, SERVICE, ENV, definition)); + current = clients.get_or_create_metrics_logs( + SERVICE, + ENV, + &instance, + &runtime_metadata, + Config::default, + Vec::new(), + ); + + if revision > 0 { + assert!( + previous.lock_or_panic().is_none(), + "a changed definition should synchronously retire the old worker" + ); + } + let worker = current + .lock_or_panic() + .as_ref() + .expect("replacement runtime worker") + .worker + .clone(); + assert_eq!( + metric_context_count(&worker).await, + 1, + "each replacement replays only the latest definition for each name" + ); + } + + let definitions = clients.registered_metrics(&instance, SERVICE, ENV); + assert_eq!(definitions.len(), 1); + assert_eq!( + definitions[0].metric_type, + libdd_telemetry::data::metrics::MetricType::Gauge + ); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn full_metric_scope_preserves_existing_definitions() { + let clients = MetricsLogsClientSet::with_registration_limit(2); + let instance = InstanceId::new("session", "runtime"); + let server = MockServer::start_async().await; + let gauge_metric = server + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("\"metric\":\"one\"") + .body_includes("\"type\":\"gauge\""); + then.status(202); + }) + .await; + clients.get_or_create_metrics_logs( + "service", + "env", + &instance, + &RuntimeMetadata::new("php", "8.3", "test"), + || test_config(&server), + Vec::new(), + ); + + assert!(clients.register_metric(&instance, "service", "env", metric("one"))); + assert!(clients.register_metric(&instance, "service", "env", metric("two"))); + let mut updated_metric = metric("one"); + updated_metric.metric_type = libdd_telemetry::data::metrics::MetricType::Gauge; + assert!(clients.register_metric(&instance, "service", "env", updated_metric)); + assert!(!clients.register_metric(&instance, "service", "env", metric("three"))); + let names = clients.registered_metric_names(&instance, "service", "env"); + assert_eq!(names, HashSet::from(["one".to_string(), "two".to_string()])); + let client = clients.get_or_create_metrics_logs( + "service", + "env", + &instance, + &RuntimeMetadata::new("php", "8.3", "test"), + || test_config(&server), + Vec::new(), + ); + let point = client + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .to_telemetry_point(("one".to_string(), 1.0, Vec::new())) + .expect("updated metric should produce a point"); + let worker = client + .lock_or_panic() + .as_ref() + .expect("runtime worker") + .worker + .clone(); + worker.send_msg(point).await.unwrap(); + worker + .send_msg(TelemetryActions::Lifecycle( + LifecycleAction::FlushMetricAggr, + )) + .await + .unwrap(); + worker + .send_msg(TelemetryActions::Lifecycle(LifecycleAction::FlushData)) + .await + .unwrap(); + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .unwrap(); + rx.await.unwrap(); + timeout(Duration::from_secs(5), async { + while gauge_metric.calls_async().await != 1 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("updated metric should be delivered as a gauge"); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metrics_logs_cache_is_scoped_by_instance() { + const SERVICE: &str = "shared-service"; + const ENV: &str = "test"; + + let server_a = MockServer::start_async().await; + let server_b = MockServer::start_async().await; + let expected_a = server_a + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("owner-a"); + then.status(202); + }) + .await; + let unexpected_a = server_a + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("owner-b"); + then.status(202); + }) + .await; + let expected_b = server_b + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("owner-b"); + then.status(202); + }) + .await; + let unexpected_b = server_b + .mock_async(|when, then| { + when.method(POST) + .path(TELEMETRY_PATH) + .body_includes("owner-a"); + then.status(202); + }) + .await; + + let sidecar = SidecarServer::default(); + let instance_a = InstanceId::new("session-a", "runtime-a"); + let instance_b = InstanceId::new("session-b", "runtime-b"); + *sidecar + .get_session(&instance_a.session_id) + .session_config + .lock_or_panic() = Some(test_config(&server_a)); + *sidecar + .get_session(&instance_b.session_id) + .session_config + .lock_or_panic() = Some(test_config(&server_b)); + sidecar.get_runtime(&instance_a); + sidecar.get_runtime(&instance_b); + + let client_a = + get_telemetry_client(&sidecar, &instance_a, SERVICE, ENV).expect("first owner"); + let client_b = + get_telemetry_client(&sidecar, &instance_b, SERVICE, ENV).expect("second owner"); + assert!(!Arc::ptr_eq(&client_a, &client_b)); + + for (instance_id, client, message) in [ + (&instance_a, &client_a, "owner-a"), + (&instance_b, &client_b, "owner-b"), + ] { + let worker = client + .lock_or_panic() + .as_ref() + .expect("metrics/logs client") + .worker + .clone(); + let delivery = TelemetryBatch::Fresh(DirectTelemetryActions { + generation: sidecar.direct_telemetry_lifecycles.generation(instance_id), + actions: InternalTelemetryActions { + instance_id: instance_id.clone(), + service_name: SERVICE.to_string(), + env_name: ENV.to_string(), + actions: vec![internal_log(message)], + }, + }) + .deliver(&sidecar) + .await; + assert!(delivery.is_ok()); + worker + .send_msg(TelemetryActions::Lifecycle(LifecycleAction::FlushData)) + .await + .unwrap(); + let (tx, rx) = futures::channel::oneshot::channel(); + worker + .send_msg(TelemetryActions::CollectStats(tx)) + .await + .unwrap(); + rx.await.unwrap(); + } + + timeout(Duration::from_secs(5), async { + while expected_a.calls_async().await != 1 || expected_b.calls_async().await != 1 { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("each owner should deliver to its own endpoint"); + assert_eq!(unexpected_a.calls_async().await, 0); + assert_eq!(unexpected_b.calls_async().await, 0); + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn metrics_logs_replay_is_scoped_by_service() { + const ENV: &str = "test"; + const SHARED_METRIC: &str = "shared.metric"; + + let http_server = MockServer::start_async().await; + let clients = MetricsLogsClientSet::default(); + let runtime_metadata = RuntimeMetadata::new("php", "8.3", "test"); + let instance_id = InstanceId::new("session", "runtime"); + + let service_a = clients.get_or_create_metrics_logs( + "service-a", + ENV, + &instance_id, + &runtime_metadata, + || test_config(&http_server), + Vec::new(), + ); + let service_b = clients.get_or_create_metrics_logs( + "service-b", + ENV, + &instance_id, + &runtime_metadata, + || test_config(&http_server), + Vec::new(), + ); + for (service, unique_metric, metric_type) in [ + ( + "service-a", + "service_a.metric", + libdd_telemetry::data::metrics::MetricType::Count, + ), + ( + "service-b", + "service_b.metric", + libdd_telemetry::data::metrics::MetricType::Gauge, + ), + ] { + for name in [SHARED_METRIC, unique_metric] { + clients.register_metric( + &instance_id, + service, + ENV, + MetricContext { + name: name.to_string(), + tags: Vec::new(), + metric_type, + common: true, + namespace: libdd_telemetry::data::metrics::MetricNamespace::Tracers, + }, + ); + } + } + assert_eq!( + clients.registered_metric_names(&instance_id, "service-a", ENV), + HashSet::from([SHARED_METRIC.to_string(), "service_a.metric".to_string(),]) + ); + assert_eq!( + clients.registered_metric_names(&instance_id, "service-b", ENV), + HashSet::from([SHARED_METRIC.to_string(), "service_b.metric".to_string(),]) + ); + + clients.remove_metrics_logs_client(&instance_id, "service-a", ENV, &service_a); + clients.remove_metrics_logs_client(&instance_id, "service-b", ENV, &service_b); + let replacement_a = clients.get_or_create_metrics_logs( + "service-a", + ENV, + &instance_id, + &runtime_metadata, + || test_config(&http_server), + Vec::new(), + ); + let replacement_b = clients.get_or_create_metrics_logs( + "service-b", + ENV, + &instance_id, + &runtime_metadata, + || test_config(&http_server), + Vec::new(), + ); + { + let replacement_a = replacement_a.lock_or_panic(); + let a_metrics = &replacement_a + .as_ref() + .expect("service A replacement") + .telemetry_metrics; + assert!(a_metrics.contains_key(SHARED_METRIC)); + assert!(a_metrics.contains_key("service_a.metric")); + assert!(!a_metrics.contains_key("service_b.metric")); + } + { + let replacement_b = replacement_b.lock_or_panic(); + let b_metrics = &replacement_b + .as_ref() + .expect("service B replacement") + .telemetry_metrics; + assert!(b_metrics.contains_key(SHARED_METRIC)); + assert!(b_metrics.contains_key("service_b.metric")); + assert!(!b_metrics.contains_key("service_a.metric")); + } + } +}