From 53da5225f50eea59460547cdf8da3766272185e7 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 11 Sep 2025 13:06:51 +0200 Subject: [PATCH 01/20] Scrub spans v2 --- relay-server/src/processing/spans/mod.rs | 8 +- relay-server/src/processing/spans/process.rs | 1364 ++++++++++++++++- ...tests__scrub_span_pii_string_rules-10.snap | 27 + ...tests__scrub_span_pii_string_rules-11.snap | 27 + ...tests__scrub_span_pii_string_rules-12.snap | 27 + ...tests__scrub_span_pii_string_rules-13.snap | 27 + ...tests__scrub_span_pii_string_rules-14.snap | 27 + ...tests__scrub_span_pii_string_rules-15.snap | 27 + ...tests__scrub_span_pii_string_rules-16.snap | 27 + ...tests__scrub_span_pii_string_rules-17.snap | 27 + ...tests__scrub_span_pii_string_rules-18.snap | 27 + ...tests__scrub_span_pii_string_rules-19.snap | 27 + ..._tests__scrub_span_pii_string_rules-2.snap | 27 + ...tests__scrub_span_pii_string_rules-20.snap | 27 + ...tests__scrub_span_pii_string_rules-21.snap | 27 + ...tests__scrub_span_pii_string_rules-22.snap | 27 + ...tests__scrub_span_pii_string_rules-23.snap | 27 + ...tests__scrub_span_pii_string_rules-24.snap | 27 + ...tests__scrub_span_pii_string_rules-25.snap | 27 + ...tests__scrub_span_pii_string_rules-26.snap | 27 + ...tests__scrub_span_pii_string_rules-27.snap | 27 + ...tests__scrub_span_pii_string_rules-28.snap | 27 + ...tests__scrub_span_pii_string_rules-29.snap | 27 + ..._tests__scrub_span_pii_string_rules-3.snap | 27 + ...tests__scrub_span_pii_string_rules-30.snap | 27 + ...tests__scrub_span_pii_string_rules-31.snap | 27 + ...tests__scrub_span_pii_string_rules-32.snap | 27 + ...tests__scrub_span_pii_string_rules-33.snap | 27 + ...tests__scrub_span_pii_string_rules-34.snap | 27 + ...tests__scrub_span_pii_string_rules-35.snap | 27 + ...tests__scrub_span_pii_string_rules-36.snap | 27 + ...tests__scrub_span_pii_string_rules-37.snap | 27 + ...tests__scrub_span_pii_string_rules-38.snap | 27 + ...tests__scrub_span_pii_string_rules-39.snap | 27 + ..._tests__scrub_span_pii_string_rules-4.snap | 27 + ...tests__scrub_span_pii_string_rules-40.snap | 27 + ...tests__scrub_span_pii_string_rules-41.snap | 27 + ...tests__scrub_span_pii_string_rules-42.snap | 27 + ...tests__scrub_span_pii_string_rules-43.snap | 27 + ...tests__scrub_span_pii_string_rules-44.snap | 27 + ...tests__scrub_span_pii_string_rules-45.snap | 27 + ...tests__scrub_span_pii_string_rules-46.snap | 27 + ...tests__scrub_span_pii_string_rules-47.snap | 27 + ...tests__scrub_span_pii_string_rules-48.snap | 27 + ...tests__scrub_span_pii_string_rules-49.snap | 27 + ..._tests__scrub_span_pii_string_rules-5.snap | 27 + ...tests__scrub_span_pii_string_rules-50.snap | 27 + ..._tests__scrub_span_pii_string_rules-6.snap | 27 + ..._tests__scrub_span_pii_string_rules-7.snap | 27 + ..._tests__scrub_span_pii_string_rules-8.snap | 27 + ..._tests__scrub_span_pii_string_rules-9.snap | 27 + ...s__tests__scrub_span_pii_string_rules.snap | 27 + relay-spans/src/v2_to_v1.rs | 1 - 53 files changed, 2719 insertions(+), 4 deletions(-) create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap create mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap diff --git a/relay-server/src/processing/spans/mod.rs b/relay-server/src/processing/spans/mod.rs index e4168e7724f..f083153404f 100644 --- a/relay-server/src/processing/spans/mod.rs +++ b/relay-server/src/processing/spans/mod.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use relay_event_normalization::GeoIpLookup; use relay_event_schema::processor::ProcessingAction; use relay_event_schema::protocol::SpanV2; +use relay_pii::PiiConfigError; use relay_quotas::{DataCategory, RateLimits}; use crate::Envelope; @@ -35,6 +36,9 @@ pub enum Error { /// A processor failed to process the spans. #[error("envelope processor failed")] ProcessingFailed(#[from] ProcessingAction), + /// Internal error, Pii config could not be loaded. + #[error("Pii configuration error")] + PiiConfig(PiiConfigError), /// The span is invalid. #[error("invalid: {0}")] Invalid(DiscardReason), @@ -51,6 +55,7 @@ impl OutcomeError for Error { let reason_code = limits.longest().and_then(|limit| limit.reason_code.clone()); Some(Outcome::RateLimited(reason_code)) } + Self::PiiConfig(_) => Some(Outcome::Invalid(DiscardReason::ProjectStatePii)), Self::ProcessingFailed(_) => Some(Outcome::Invalid(DiscardReason::Internal)), Self::Invalid(reason) => Some(Outcome::Invalid(*reason)), }; @@ -126,11 +131,10 @@ impl processing::Processor for SpansProcessor { process::normalize(&mut spans, &self.geo_lookup); filter::filter(&mut spans, ctx); + process::scrub(&mut spans, ctx); self.limiter.enforce_quotas(&mut spans, ctx).await?; - // TODO: pii scrubbing - let metrics = dynamic_sampling::create_indexed_metrics(&spans, ctx); Ok(Output { diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 7787f7a60ef..ff44fe8176e 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -1,13 +1,15 @@ use relay_event_normalization::{ GeoIpLookup, SchemaProcessor, TimestampProcessor, TrimmingProcessor, eap, }; -use relay_event_schema::processor::{ProcessingState, process_value}; +use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::SpanV2; +use relay_pii::{AttributeMode, PiiProcessor}; use relay_protocol::Annotated; use crate::envelope::{ContainerItems, Item, ItemContainer}; use crate::extractors::RequestMeta; use crate::managed::Managed; +use crate::processing::Context; use crate::processing::spans::{Error, ExpandedSpans, Result, SampledSpans}; use crate::services::outcome::DiscardReason; @@ -82,3 +84,1363 @@ fn normalize_span( Ok(()) } + +/// Applies PII scrubbing to individual spans. +pub fn scrub(spans: &mut Managed, ctx: Context<'_>) { + spans.modify(|spans, records| { + spans.spans.retain_mut(|span| { + let r = scrub_span(span, ctx).inspect_err(|err| { + relay_log::debug!("failed to scrub pii from span: {err}"); + }); + + records.or_default(r.map(|_| true), &*span) + }) + }); +} + +fn scrub_span(span: &mut Annotated, ctx: Context<'_>) -> Result<()> { + let pii_config_from_scrubbing = ctx + .project_info + .config + .datascrubbing_settings + .pii_config() + .map_err(|e| Error::PiiConfig(e.clone()))?; + + let state = ProcessingState::root().enter_borrowed("", None, [ValueType::Span]); + + if let Some(ref config) = ctx.project_info.config.pii_config { + let mut processor = PiiProcessor::new(config.compiled()) + // For advanced rules we want to treat attributes as objects. + .attribute_mode(AttributeMode::Object); + process_value(span, &mut processor, &state)?; + } + + if let Some(config) = pii_config_from_scrubbing { + let mut processor = PiiProcessor::new(config.compiled()) + // For "legacy" rules we want to identify attributes with their values. + .attribute_mode(AttributeMode::ValueOnly); + process_value(span, &mut processor, &state)?; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use relay_pii::{DataScrubbingConfig, PiiConfig}; + use relay_protocol::SerializableAnnotated; + + use crate::services::projects::project::ProjectInfo; + + use super::*; + + fn make_context( + scrubbing_config: DataScrubbingConfig, + pii_config: Option, + ) -> Context<'static> { + let config = Box::leak(Box::new(relay_config::Config::default())); + let global_config = Box::leak(Box::new(relay_dynamic_config::GlobalConfig::default())); + let project_info = Box::leak(Box::new(ProjectInfo { + config: relay_dynamic_config::ProjectConfig { + pii_config, + datascrubbing_settings: scrubbing_config, + ..Default::default() + }, + ..Default::default() + })); + let rate_limits = Box::leak(Box::new(relay_quotas::RateLimits::default())); + + Context { + config, + global_config, + project_info, + rate_limits, + sampling_project_info: None, + } + } + + #[test] + fn test_scrub_span_pii_default_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `user.name` is the only one that should be scrubbed by default rules. + let json = r#"{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "secret123" + }, + "secret": { + "type": "string", + "value": "topsecret" + }, + "api_key": { + "type": "string", + "value": "sk-1234567890abcdef" + }, + "auth_token": { + "type": "string", + "value": "bearer_token_123" + }, + "credit_card": { + "type": "string", + "value": "4571234567890111" + }, + "visa_card": { + "type": "string", + "value": "4571 2345 6789 0111" + }, + "bank_account": { + "type": "string", + "value": "DE89370400440532013000" + }, + "iban_code": { + "type": "string", + "value": "NO9386011117945" + }, + "authorization": { + "type": "string", + "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + }, + "private_key": { + "type": "string", + "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" + }, + "database_url": { + "type": "string", + "value": "https://username:password@example.com/db" + }, + "file_path": { + "type": "string", + "value": "C:\\Users\\john\\Documents\\secret.txt" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "social_security": { + "type": "string", + "value": "078-05-1120" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "client_ip": { + "type": "string", + "value": "192.168.1.100" + }, + "ipv6_addr": { + "type": "string", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "very_sensitive_data": { + "type": "string", + "value": "very_sensitive_data" + } + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = true; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ]; + scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; + + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "api_key": { + "type": "string", + "value": "[Filtered]" + }, + "auth_token": { + "type": "string", + "value": "[Filtered]" + }, + "authorization": { + "type": "string", + "value": "[Filtered]" + }, + "bank_account": { + "type": "string", + "value": "[Filtered]" + }, + "client_ip": { + "type": "string", + "value": "[ip]" + }, + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "database_url": { + "type": "string", + "value": "[Filtered]" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "file_path": { + "type": "string", + "value": "[Filtered]" + }, + "iban_code": { + "type": "string", + "value": "[Filtered]" + }, + "ipv6_addr": { + "type": "string", + "value": "[ip]" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "private_key": { + "type": "string", + "value": "[Filtered]" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "secret": { + "type": "string", + "value": "[Filtered]" + }, + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "social_security": { + "type": "string", + "value": "[Filtered]" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "very_sensitive_data": { + "type": "string", + "value": "[Filtered]" + }, + "visa_card": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "api_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "auth_token": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "authorization": { + "value": { + "": { + "rem": [ + [ + "@bearer:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + }, + "bank_account": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 22 + } + } + }, + "client_ip": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 13 + } + } + }, + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "database_url": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 40 + } + } + }, + "file_path": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 34 + } + } + }, + "iban_code": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + }, + "ipv6_addr": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 39 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "private_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 118 + } + } + }, + "secret": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "social_security": { + "value": { + "": { + "rem": [ + [ + "@usssn:filter", + "s", + 0, + 10 + ] + ], + "len": 11 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "very_sensitive_data": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "visa_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_span_pii_custom_object_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + }, + "project:4": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[USER NAME]" + } + }, + "project:5": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[RELEASE]" + } + }, + "project:6": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[URL PATH]" + } + }, + "project:7": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[DESCRIPTION]" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ], + "'user.name'.value": [ + "project:4" + ], + "'sentry.release'.value": [ + "project:5" + ], + "'url.path'.value": [ + "project:6" + ], + "'sentry.description'.value": [ + "project:7" + ] + } + } + )) + .unwrap(); + + let ctx = make_context(scrubbing_config, Some(config)); + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "sentry.description": { + "type": "string", + "value": "[DESCRIPTION]" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "url.path": { + "type": "string", + "value": "[URL PATH]" + }, + "user.name": { + "type": "string", + "value": "[USER NAME]" + }, + "_meta": { + "sentry.description": { + "value": { + "": { + "rem": [ + [ + "project:7", + "s", + 0, + 13 + ] + ], + "len": 9 + } + } + }, + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + }, + "url.path": { + "value": { + "": { + "rem": [ + [ + "project:6", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "project:4", + "s", + 0, + 11 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_span_pii_string_rules() { + let test_cases = vec![ + // IP rules + ("@ip", "127.0.0.1"), + ("@ip:replace", "127.0.0.1"), + ("@ip:remove", "127.0.0.1"), + ("@ip:mask", "127.0.0.1"), + // Email rules + ("@email", "test@example.com"), + ("@email:replace", "test@example.com"), + ("@email:remove", "test@example.com"), + ("@email:mask", "test@example.com"), + // Credit card rules + ("@creditcard", "4242424242424242"), + ("@creditcard:replace", "4242424242424242"), + ("@creditcard:remove", "4242424242424242"), + ("@creditcard:mask", "4242424242424242"), + // IBAN rules + ("@iban", "DE89370400440532013000"), + ("@iban:replace", "DE89370400440532013000"), + ("@iban:remove", "DE89370400440532013000"), + ("@iban:mask", "DE89370400440532013000"), + // MAC address rules + ("@mac", "4a:00:04:10:9b:50"), + ("@mac:replace", "4a:00:04:10:9b:50"), + ("@mac:remove", "4a:00:04:10:9b:50"), + ("@mac:mask", "4a:00:04:10:9b:50"), + // UUID rules + ("@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + ("@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), + // IMEI rules + ("@imei", "356938035643809"), + ("@imei:replace", "356938035643809"), + ("@imei:remove", "356938035643809"), + // PEM key rules + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + ( + "@pemkey:replace", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + ( + "@pemkey:remove", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + ), + // URL auth rules + ("@urlauth", "https://username:password@example.com/"), + ("@urlauth:replace", "https://username:password@example.com/"), + ("@urlauth:remove", "https://username:password@example.com/"), + ("@urlauth:mask", "https://username:password@example.com/"), + // US SSN rules + ("@usssn", "078-05-1120"), + ("@usssn:replace", "078-05-1120"), + ("@usssn:remove", "078-05-1120"), + ("@usssn:mask", "078-05-1120"), + // User path rules + ("@userpath", "/Users/john/Documents"), + ("@userpath:replace", "/Users/john/Documents"), + ("@userpath:remove", "/Users/john/Documents"), + ("@userpath:mask", "/Users/john/Documents"), + // Password rules + ("@password", "my_password_123"), // @password defaults to remove + ("@password:remove", "my_password_123"), + ("@password:replace", "my_password_123"), + ("@password:mask", "my_password_123"), + // Bearer token rules + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"), + ( + "@bearer:replace", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ( + "@bearer:remove", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ( + "@bearer:mask", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + ), + ]; + + for (rule_type, test_value) in test_cases { + let json = format!( + r#"{{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = rule_type, + value = test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut data = Annotated::::from_json(&json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": [rule_type] + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes)); + } + } + + #[test] + fn test_scrub_span_implicit_attribute_value_does_not_match() { + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "$span.attributes.remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "###); + } + + #[test] + fn test_scrub_span_explicit_attribute_value_does_match() { + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "$span.attributes.remove_this_string_abc123.value": ["remove_abc123"], + } + })) + .unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + + let ctx = make_context(scrubbing_config, Some(config)); + + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "attributes": { + "remove_this_string_abc123": { + "type": "string", + "value": "abc---" + } + }, + "_meta": { + "attributes": { + "remove_this_string_abc123": { + "value": { + "": { + "rem": [ + [ + "remove_abc123", + "s", + 0, + 6 + ] + ], + "len": 6 + } + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_span_sensitive_fields() { + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "should_be_removed" + }, + "another_sensitive": { + "type": "integer", + "value": 42 + }, + "my_value": { + "type": "string", + "value": "this_should_be_removed_as_sensitive" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "sensitive_custom".to_owned(), + "another_sensitive".to_owned(), + ]; + + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "another_sensitive": { + "type": "integer", + "value": null + }, + "my_value": { + "type": "string", + "value": "[Filtered]" + }, + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "another_sensitive": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "x" + ] + ] + } + } + }, + "my_value": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 35 + } + } + }, + "sensitive_custom": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 17 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_span_safe_fields() { + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": { + "password": { + "type": "string", + "value": "secret123" + }, + "credit_card": { + "type": "string", + "value": "4242424242424242" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + } + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = false; + scrubbing_config.exclude_fields = vec!["secret".to_owned()]; // Only 'secret' is safe + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + { + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + }, + "_meta": { + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap new file mode 100644 index 00000000000..5e0644e8858 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:replace|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard:replace|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:replace", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap new file mode 100644 index 00000000000..d6fcf4eeed4 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:remove|4242424242424242": { + "type": "string", + "value": "" + }, + "_meta": { + "@creditcard:remove|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap new file mode 100644 index 00000000000..83b5941e72b --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard:mask|4242424242424242": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@creditcard:mask|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap new file mode 100644 index 00000000000..0098214af88 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap new file mode 100644 index 00000000000..8e7180f1fb8 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:replace|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban:replace|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:replace", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap new file mode 100644 index 00000000000..e4e4f157cc7 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:remove|DE89370400440532013000": { + "type": "string", + "value": "" + }, + "_meta": { + "@iban:remove|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:remove", + "x", + 0, + 0 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap new file mode 100644 index 00000000000..6e9e9a41591 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@iban:mask|DE89370400440532013000": { + "type": "string", + "value": "**********************" + }, + "_meta": { + "@iban:mask|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:mask", + "m", + 0, + 22 + ] + ], + "len": 22 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap new file mode 100644 index 00000000000..1bb20ef1177 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap new file mode 100644 index 00000000000..2252f0e360b --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:replace|4a:00:04:10:9b:50": { + "type": "string", + "value": "[mac]" + }, + "_meta": { + "@mac:replace|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:replace", + "s", + 0, + 5 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap new file mode 100644 index 00000000000..b8eae592a5e --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:remove|4a:00:04:10:9b:50": { + "type": "string", + "value": "" + }, + "_meta": { + "@mac:remove|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:remove", + "x", + 0, + 0 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap new file mode 100644 index 00000000000..511c45134a8 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:replace|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip:replace|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap new file mode 100644 index 00000000000..eb85d44e55e --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@mac:mask|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac:mask|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:mask", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap new file mode 100644 index 00000000000..3985d61bac0 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap new file mode 100644 index 00000000000..0e462295fdf --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "[uuid]" + }, + "_meta": { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:replace", + "s", + 0, + 6 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap new file mode 100644 index 00000000000..bd65b71cc17 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "" + }, + "_meta": { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:remove", + "x", + 0, + 0 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap new file mode 100644 index 00000000000..e9e60427027 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:mask", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap new file mode 100644 index 00000000000..56b94c00632 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap new file mode 100644 index 00000000000..54d8ffc74ea --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei:replace|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei:replace|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:replace", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap new file mode 100644 index 00000000000..b2082a9a4ca --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@imei:remove|356938035643809": { + "type": "string", + "value": "" + }, + "_meta": { + "@imei:remove|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap new file mode 100644 index 00000000000..01619327b8e --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap new file mode 100644 index 00000000000..1bc4eb0e266 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:replace", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap new file mode 100644 index 00000000000..0af041ca2d7 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:remove|127.0.0.1": { + "type": "string", + "value": "" + }, + "_meta": { + "@ip:remove|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:remove", + "x", + 0, + 0 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap new file mode 100644 index 00000000000..6ffe7daf5bb --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:remove", + "x", + 31, + 31 + ] + ], + "len": 124 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap new file mode 100644 index 00000000000..680380ffe6d --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap new file mode 100644 index 00000000000..e7ef4496618 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:replace|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth:replace|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:replace", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap new file mode 100644 index 00000000000..bb838483d5f --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:remove|https://username:password@example.com/": { + "type": "string", + "value": "https://@example.com/" + }, + "_meta": { + "@urlauth:remove|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:remove", + "x", + 8, + 8 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap new file mode 100644 index 00000000000..19221198ab7 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@urlauth:mask|https://username:password@example.com/": { + "type": "string", + "value": "https://*****************@example.com/" + }, + "_meta": { + "@urlauth:mask|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:mask", + "m", + 8, + 25 + ] + ], + "len": 38 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap new file mode 100644 index 00000000000..a95c6a9d5fa --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap new file mode 100644 index 00000000000..f0b410d8b0c --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:replace|078-05-1120": { + "type": "string", + "value": "[us-ssn]" + }, + "_meta": { + "@usssn:replace|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:replace", + "s", + 0, + 8 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap new file mode 100644 index 00000000000..823aa122201 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:remove|078-05-1120": { + "type": "string", + "value": "" + }, + "_meta": { + "@usssn:remove|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:remove", + "x", + 0, + 0 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap new file mode 100644 index 00000000000..a7713e60da8 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@usssn:mask|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn:mask|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:mask", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap new file mode 100644 index 00000000000..726bafeb0b2 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap new file mode 100644 index 00000000000..188963150d0 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip:mask|127.0.0.1": { + "type": "string", + "value": "*********" + }, + "_meta": { + "@ip:mask|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:mask", + "m", + 0, + 9 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap new file mode 100644 index 00000000000..22b8635d29b --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:replace|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath:replace|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:replace", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap new file mode 100644 index 00000000000..c8b4e7eb704 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:remove|/Users/john/Documents": { + "type": "string", + "value": "/Users//Documents" + }, + "_meta": { + "@userpath:remove|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:remove", + "x", + 7, + 7 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap new file mode 100644 index 00000000000..22717e2fd8c --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@userpath:mask|/Users/john/Documents": { + "type": "string", + "value": "/Users/****/Documents" + }, + "_meta": { + "@userpath:mask|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:mask", + "m", + 7, + 11 + ] + ], + "len": 21 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap new file mode 100644 index 00000000000..a09ff5b5ccd --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap new file mode 100644 index 00000000000..c9acdca4b62 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:remove|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password:remove|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap new file mode 100644 index 00000000000..5c9917bbd01 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:replace|my_password_123": { + "type": "string", + "value": "[password]" + }, + "_meta": { + "@password:replace|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:replace", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap new file mode 100644 index 00000000000..277805125cc --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@password:mask|my_password_123": { + "type": "string", + "value": "***************" + }, + "_meta": { + "@password:mask|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:mask", + "m", + 0, + 15 + ] + ], + "len": 15 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap new file mode 100644 index 00000000000..67f8b7f6852 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap new file mode 100644 index 00000000000..832eae1a17f --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:replace", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap new file mode 100644 index 00000000000..4fe5634c249 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "" + }, + "_meta": { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:remove", + "x", + 0, + 0 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap new file mode 100644 index 00000000000..5fbb49273d1 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap new file mode 100644 index 00000000000..f66f31b45b6 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "*******************************************" + }, + "_meta": { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:mask", + "m", + 0, + 43 + ] + ], + "len": 43 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap new file mode 100644 index 00000000000..1e09eda548a --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:replace|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email:replace|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:replace", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap new file mode 100644 index 00000000000..bc35eb53873 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:remove|test@example.com": { + "type": "string", + "value": "" + }, + "_meta": { + "@email:remove|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap new file mode 100644 index 00000000000..75dd60d6717 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@email:mask|test@example.com": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@email:mask|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap new file mode 100644 index 00000000000..5f2fe545179 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@creditcard|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } +} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap new file mode 100644 index 00000000000..67d74a47d45 --- /dev/null +++ b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap @@ -0,0 +1,27 @@ +--- +source: relay-server/src/processing/spans/process.rs +expression: SerializableAnnotated(&data.value().unwrap().attributes) +--- +{ + "@ip|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } +} diff --git a/relay-spans/src/v2_to_v1.rs b/relay-spans/src/v2_to_v1.rs index 27a4093c748..a20f92d22ec 100644 --- a/relay-spans/src/v2_to_v1.rs +++ b/relay-spans/src/v2_to_v1.rs @@ -23,7 +23,6 @@ use url::Url; /// inferred from other attributes if the `sentry.op` attribute is not set. /// * The V1 span's `description` field will be set based on the V2 span's `sentry.description` /// attribute, or inferred from other attributes if the `sentry.description` attribute is not set. -/// * The V1 span's `description` field is set based on the V2 span's `sentry.description` attribute. /// * The V1 span's `status` field is set based on the V2 span's `status` field and /// `http.status_code` and `rpc.grpc.status_code` attributes. /// * The V1 span's `exclusive_time` field is set based on the V2 span's `exclusive_time_nano` From b56a3557d5480368fd87e00adb4f05bc37ed9be6 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Thu, 11 Sep 2025 13:28:57 +0200 Subject: [PATCH 02/20] Port tests from logs --- tests/integration/test_ourlogs.py | 1 - tests/integration/test_spansv2.py | 148 +++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 1f0aa5fcae9..717e074beb6 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -330,7 +330,6 @@ def test_ourlog_extraction_with_sentry_logs( def test_ourlog_extraction_with_string_pii_scrubbing( mini_sentry, relay, - items_consumer, rule_type, test_value, expected_scrubbed, diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index 7d326e6f1ae..f8d17577f32 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from unittest import mock from sentry_sdk.envelope import Envelope, Item, PayloadRef @@ -8,6 +8,7 @@ from .test_dynamic_sampling import _add_sampling_config +import json import pytest TEST_CONFIG = { @@ -557,3 +558,148 @@ def test_spanv2_inbound_filters( ] assert mini_sentry.captured_events.empty() + + +@pytest.mark.parametrize( + "rule_type,test_value,expected_scrubbed", + [ + ("@ip", "127.0.0.1", "[ip]"), + ("@email", "test@example.com", "[email]"), + ("@creditcard", "4242424242424242", "[creditcard]"), + ("@iban", "DE89370400440532013000", "[iban]"), + ("@mac", "4a:00:04:10:9b:50", "*****************"), + ( + "@uuid", + "ceee0822-ed8f-4622-b2a3-789e73e75cd1", + "************************************", + ), + ("@imei", "356938035643809", "[imei]"), + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", + ), + ( + "@urlauth", + "https://username:password@example.com/", + "https://[auth]@example.com/", + ), + ("@usssn", "078-05-1120", "***********"), + ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), + ("@password", "my_password_123", ""), + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), + ], +) +def test_spanv2_with_string_pii_scrubbing( + mini_sentry, + relay, + rule_type, + test_value, + expected_scrubbed, +): + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:standalone-span-ingestion", + "projects:span-v2-experimental-processing", + ] + + project_config["config"]["piiConfig"]["applications"] = {"$string": [rule_type]} + + relay = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp() + 0.5, + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "Test span", + "is_remote": False, + "attributes": { + "test_pii": {"value": test_value, "type": "string"}, + }, + } + ) + + relay.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + attributes = item["attributes"] + + assert "test_pii" in attributes + assert attributes["test_pii"]["value"] == expected_scrubbed + assert "_meta" in item + meta = item["_meta"]["attributes"]["test_pii"]["value"][""] + assert "rem" in meta + + # Check that the rule type is mentioned in the metadata + rem_info = meta["rem"][0] + assert rule_type in rem_info[0] + + +@pytest.mark.parametrize( + "attribute_key,attribute_value,expected_value,rule_type", + [ + ("password", "my_password_123", "[Filtered]", "@password:filter"), + ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), + ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), + ], +) +def test_spanv2_default_pii_scrubbing_attributes( + mini_sentry, + relay, + attribute_key, + attribute_value, + expected_value, + rule_type, +): + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:standalone-span-ingestion", + "projects:span-v2-experimental-processing", + ] + project_config["config"].setdefault( + "datascrubbingSettings", + { + "scrubData": True, + "scrubDefaults": True, + "scrubIpAddresses": True, + }, + ) + + relay_instance = relay(mini_sentry, options=TEST_CONFIG) + ts = datetime.now(timezone.utc) + + envelope = envelope_with_spans( + { + "start_timestamp": ts.timestamp() + 0.5, + "end_timestamp": ts.timestamp() + 0.5, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "Test span", + "attributes": { + attribute_key: {"value": attribute_value, "type": "string"}, + }, + } + ) + + relay_instance.send_envelope(project_id, envelope) + + envelope = mini_sentry.captured_events.get() + item_payload = json.loads(envelope.items[0].payload.bytes.decode()) + item = item_payload["items"][0] + attributes = item["attributes"] + + assert attribute_key in attributes + assert attributes[attribute_key]["value"] == expected_value + assert "_meta" in item + meta = item["_meta"]["attributes"][attribute_key]["value"][""] + assert "rem" in meta + rem_info = meta["rem"] + assert len(rem_info) == 1 + assert rem_info[0][0] == rule_type From c0994b27fa3b4a8dcd516629f195ccb7bb3b91eb Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 19 Sep 2025 15:42:25 +0200 Subject: [PATCH 03/20] Drive-by doc fix --- relay-config/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay-config/src/config.rs b/relay-config/src/config.rs index b43e16a54fc..50c662df0f4 100644 --- a/relay-config/src/config.rs +++ b/relay-config/src/config.rs @@ -622,7 +622,7 @@ pub struct Limits { pub max_api_chunk_upload_size: ByteSize, /// The maximum payload size for a profile pub max_profile_size: ByteSize, - /// The maximum payload size for a span. + /// The maximum payload size for a log. pub max_log_size: ByteSize, /// The maximum payload size for a span. pub max_span_size: ByteSize, From e924d992c5cc6f15bcdad90e6102a68ebc74a7c5 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 19 Sep 2025 15:58:30 +0200 Subject: [PATCH 04/20] Add tests for span links --- relay-server/src/processing/spans/process.rs | 411 ++++++++++++++++++- 1 file changed, 408 insertions(+), 3 deletions(-) diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index ff44fe8176e..1b362d4b4e4 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -295,7 +295,9 @@ mod tests { let ctx = make_context(scrubbing_config, None); scrub_span(&mut data, ctx).unwrap(); - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" + let span = data.value().unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&span.attributes), @r###" { "api_key": { "type": "string", @@ -362,8 +364,8 @@ mod tests { "value": "[Filtered]" }, "sentry.description": { - "type": "string", - "value": "secret123" + "type": "string", + "value": "secret123" }, "sentry.release": { "type": "string", @@ -662,6 +664,92 @@ mod tests { "###); } + #[test] + fn test_scrub_span_pii_default_rules_links() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `user.name` is the only one that should be scrubbed by default rules. + let json = r#"{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "links": [{ + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + } + } + }] + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = true; + scrubbing_config.scrub_ip_addresses = true; + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ]; + scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; + + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + let link = data.value().unwrap().links.value().unwrap()[0] + .value() + .unwrap(); + insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" + { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + #[test] fn test_scrub_span_pii_custom_object_rules() { // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: @@ -974,6 +1062,323 @@ mod tests { "###); } + #[test] + fn test_scrub_span_pii_custom_object_rules_links() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. + let json = r#" + { + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "links": [{ + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + } + }] + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + }, + "project:4": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[USER NAME]" + } + }, + "project:5": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[RELEASE]" + } + }, + "project:6": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[URL PATH]" + } + }, + "project:7": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[DESCRIPTION]" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ], + "'user.name'.value": [ + "project:4" + ], + "'sentry.release'.value": [ + "project:5" + ], + "'url.path'.value": [ + "project:6" + ], + "'sentry.description'.value": [ + "project:7" + ] + } + } + )) + .unwrap(); + + let ctx = make_context(scrubbing_config, Some(config)); + scrub_span(&mut data, ctx).unwrap(); + let link = data.value().unwrap().links.value().unwrap()[0] + .value() + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "sentry.description": { + "type": "string", + "value": "[DESCRIPTION]" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "url.path": { + "type": "string", + "value": "[URL PATH]" + }, + "user.name": { + "type": "string", + "value": "[USER NAME]" + }, + "_meta": { + "sentry.description": { + "value": { + "": { + "rem": [ + [ + "project:7", + "s", + 0, + 13 + ] + ], + "len": 9 + } + } + }, + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + }, + "url.path": { + "value": { + "": { + "rem": [ + [ + "project:6", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "project:4", + "s", + 0, + 11 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + #[test] fn test_scrub_span_pii_string_rules() { let test_cases = vec![ From 8e327081a338e24d9fba31781b3e1ad2eebeaa37 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 19 Sep 2025 16:48:44 +0200 Subject: [PATCH 05/20] Lint --- tests/integration/test_spansv2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index f8d17577f32..ae9cff26220 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -1,4 +1,4 @@ -from datetime import datetime, timezone, timedelta +from datetime import datetime, timezone from unittest import mock from sentry_sdk.envelope import Envelope, Item, PayloadRef From 3aece6a68a8a3157f508d0f20450d629fc48f934 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 19 Sep 2025 16:49:37 +0200 Subject: [PATCH 06/20] Test timestamps --- tests/integration/test_spansv2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index ae9cff26220..3b902a97fbc 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -611,7 +611,7 @@ def test_spanv2_with_string_pii_scrubbing( envelope = envelope_with_spans( { - "start_timestamp": ts.timestamp() + 0.5, + "start_timestamp": ts.timestamp(), "end_timestamp": ts.timestamp() + 0.5, "trace_id": "5b8efff798038103d269b633813fc60c", "span_id": "eee19b7ec3c1b174", @@ -677,7 +677,7 @@ def test_spanv2_default_pii_scrubbing_attributes( envelope = envelope_with_spans( { - "start_timestamp": ts.timestamp() + 0.5, + "start_timestamp": ts.timestamp(), "end_timestamp": ts.timestamp() + 0.5, "trace_id": "5b8efff798038103d269b633813fc60c", "span_id": "eee19b7ec3c1b174", From d801f071af40f0822a20c7dbde9a740955f43881 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 14 Oct 2025 12:54:25 +0200 Subject: [PATCH 07/20] Inline test snapshots --- relay-server/src/processing/spans/process.rs | 1487 +++++++++++++++-- ...tests__scrub_span_pii_string_rules-10.snap | 27 - ...tests__scrub_span_pii_string_rules-11.snap | 27 - ...tests__scrub_span_pii_string_rules-12.snap | 27 - ...tests__scrub_span_pii_string_rules-13.snap | 27 - ...tests__scrub_span_pii_string_rules-14.snap | 27 - ...tests__scrub_span_pii_string_rules-15.snap | 27 - ...tests__scrub_span_pii_string_rules-16.snap | 27 - ...tests__scrub_span_pii_string_rules-17.snap | 27 - ...tests__scrub_span_pii_string_rules-18.snap | 27 - ...tests__scrub_span_pii_string_rules-19.snap | 27 - ..._tests__scrub_span_pii_string_rules-2.snap | 27 - ...tests__scrub_span_pii_string_rules-20.snap | 27 - ...tests__scrub_span_pii_string_rules-21.snap | 27 - ...tests__scrub_span_pii_string_rules-22.snap | 27 - ...tests__scrub_span_pii_string_rules-23.snap | 27 - ...tests__scrub_span_pii_string_rules-24.snap | 27 - ...tests__scrub_span_pii_string_rules-25.snap | 27 - ...tests__scrub_span_pii_string_rules-26.snap | 27 - ...tests__scrub_span_pii_string_rules-27.snap | 27 - ...tests__scrub_span_pii_string_rules-28.snap | 27 - ...tests__scrub_span_pii_string_rules-29.snap | 27 - ..._tests__scrub_span_pii_string_rules-3.snap | 27 - ...tests__scrub_span_pii_string_rules-30.snap | 27 - ...tests__scrub_span_pii_string_rules-31.snap | 27 - ...tests__scrub_span_pii_string_rules-32.snap | 27 - ...tests__scrub_span_pii_string_rules-33.snap | 27 - ...tests__scrub_span_pii_string_rules-34.snap | 27 - ...tests__scrub_span_pii_string_rules-35.snap | 27 - ...tests__scrub_span_pii_string_rules-36.snap | 27 - ...tests__scrub_span_pii_string_rules-37.snap | 27 - ...tests__scrub_span_pii_string_rules-38.snap | 27 - ...tests__scrub_span_pii_string_rules-39.snap | 27 - ..._tests__scrub_span_pii_string_rules-4.snap | 27 - ...tests__scrub_span_pii_string_rules-40.snap | 27 - ...tests__scrub_span_pii_string_rules-41.snap | 27 - ...tests__scrub_span_pii_string_rules-42.snap | 27 - ...tests__scrub_span_pii_string_rules-43.snap | 27 - ...tests__scrub_span_pii_string_rules-44.snap | 27 - ...tests__scrub_span_pii_string_rules-45.snap | 27 - ...tests__scrub_span_pii_string_rules-46.snap | 27 - ...tests__scrub_span_pii_string_rules-47.snap | 27 - ...tests__scrub_span_pii_string_rules-48.snap | 27 - ...tests__scrub_span_pii_string_rules-49.snap | 27 - ..._tests__scrub_span_pii_string_rules-5.snap | 27 - ...tests__scrub_span_pii_string_rules-50.snap | 27 - ..._tests__scrub_span_pii_string_rules-6.snap | 27 - ..._tests__scrub_span_pii_string_rules-7.snap | 27 - ..._tests__scrub_span_pii_string_rules-8.snap | 27 - ..._tests__scrub_span_pii_string_rules-9.snap | 27 - ...s__tests__scrub_span_pii_string_rules.snap | 27 - 51 files changed, 1370 insertions(+), 1467 deletions(-) delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap delete mode 100644 relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 14b044e1998..412747045d1 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -1381,135 +1381,1388 @@ mod tests { "###); } - #[test] - fn test_scrub_span_pii_string_rules() { - let test_cases = vec![ - // IP rules - ("@ip", "127.0.0.1"), - ("@ip:replace", "127.0.0.1"), - ("@ip:remove", "127.0.0.1"), - ("@ip:mask", "127.0.0.1"), - // Email rules - ("@email", "test@example.com"), - ("@email:replace", "test@example.com"), - ("@email:remove", "test@example.com"), - ("@email:mask", "test@example.com"), - // Credit card rules - ("@creditcard", "4242424242424242"), - ("@creditcard:replace", "4242424242424242"), - ("@creditcard:remove", "4242424242424242"), - ("@creditcard:mask", "4242424242424242"), - // IBAN rules - ("@iban", "DE89370400440532013000"), - ("@iban:replace", "DE89370400440532013000"), - ("@iban:remove", "DE89370400440532013000"), - ("@iban:mask", "DE89370400440532013000"), - // MAC address rules - ("@mac", "4a:00:04:10:9b:50"), - ("@mac:replace", "4a:00:04:10:9b:50"), - ("@mac:remove", "4a:00:04:10:9b:50"), - ("@mac:mask", "4a:00:04:10:9b:50"), - // UUID rules - ("@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - // IMEI rules - ("@imei", "356938035643809"), - ("@imei:replace", "356938035643809"), - ("@imei:remove", "356938035643809"), - // PEM key rules - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:replace", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:remove", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - // URL auth rules - ("@urlauth", "https://username:password@example.com/"), - ("@urlauth:replace", "https://username:password@example.com/"), - ("@urlauth:remove", "https://username:password@example.com/"), - ("@urlauth:mask", "https://username:password@example.com/"), - // US SSN rules - ("@usssn", "078-05-1120"), - ("@usssn:replace", "078-05-1120"), - ("@usssn:remove", "078-05-1120"), - ("@usssn:mask", "078-05-1120"), - // User path rules - ("@userpath", "/Users/john/Documents"), - ("@userpath:replace", "/Users/john/Documents"), - ("@userpath:remove", "/Users/john/Documents"), - ("@userpath:mask", "/Users/john/Documents"), - // Password rules - ("@password", "my_password_123"), // @password defaults to remove - ("@password:remove", "my_password_123"), - ("@password:replace", "my_password_123"), - ("@password:mask", "my_password_123"), - // Bearer token rules - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"), - ( - "@bearer:replace", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:remove", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:mask", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ]; - - for (rule_type, test_value) in test_cases { - let json = format!( - r#"{{ - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": {{ - "{rule_type}|{value}": {{ - "type": "string", - "value": "{value}" + macro_rules! attribute_rule_test { + ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => { + #[test] + fn $test_name() { + let json = format!(r#"{{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} }} - }} - }}"#, - rule_type = rule_type, - value = test_value + }}"#, + rule_type = $rule_type, + value = $test_value .replace('\\', "\\\\") .replace('"', "\\\"") .replace('\n', "\\n") - ); + ); - let mut data = Annotated::::from_json(&json).unwrap(); + let mut data = Annotated::::from_json(&json).unwrap(); - let config = serde_json::from_value::(serde_json::json!({ - "applications": { - "$string": [rule_type] - } - })) - .unwrap(); + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": [$rule_type] + } + })) + .unwrap(); - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; + let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; - let ctx = make_context(scrubbing_config, Some(config)); + let ctx = make_context(scrubbing_config, Some(config)); - scrub_span(&mut data, ctx).unwrap(); + scrub_span(&mut data, ctx).unwrap(); - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes)); + insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @$snapshot); + } + }; + } + + // IP rules + attribute_rule_test!(test_scrub_span_pii_string_rules_ip, "@ip", "127.0.0.1", @r###" + { + "@ip|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_ip_replace, "@ip:replace", "127.0.0.1", @r###" + { + "@ip:replace|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip:replace|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_ip_remove, "@ip:remove", "127.0.0.1", @r###" + { + "@ip:remove|127.0.0.1": { + "type": "string", + "value": "" + }, + "_meta": { + "@ip:remove|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:remove", + "x", + 0, + 0 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_ip_mask, "@ip:mask", "127.0.0.1", @r###" + { + "@ip:mask|127.0.0.1": { + "type": "string", + "value": "*********" + }, + "_meta": { + "@ip:mask|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:mask", + "m", + 0, + 9 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + // Email rules + attribute_rule_test!(test_scrub_span_pii_string_rules_email, "@email", "test@example.com", @r###" + { + "@email|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_email_replace, "@email:replace", "test@example.com", @r###" + { + "@email:replace|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email:replace|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:replace", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_email_remove, "@email:remove", "test@example.com", @r###" + { + "@email:remove|test@example.com": { + "type": "string", + "value": "" + }, + "_meta": { + "@email:remove|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_email_mask, "@email:mask", "test@example.com", @r###" + { + "@email:mask|test@example.com": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@email:mask|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // Credit card rules + attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard, "@creditcard", "4242424242424242", @r###" + { + "@creditcard|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_replace, "@creditcard:replace", "4242424242424242", @r###" + { + "@creditcard:replace|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard:replace|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:replace", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_remove, "@creditcard:remove", "4242424242424242", @r###" + { + "@creditcard:remove|4242424242424242": { + "type": "string", + "value": "" + }, + "_meta": { + "@creditcard:remove|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_mask, "@creditcard:mask", "4242424242424242", @r###" + { + "@creditcard:mask|4242424242424242": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@creditcard:mask|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // IBAN rules + attribute_rule_test!(test_scrub_span_pii_string_rules_iban, "@iban", "DE89370400440532013000", @r###" + { + "@iban|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_iban_replace, "@iban:replace", "DE89370400440532013000", @r###" + { + "@iban:replace|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban:replace|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:replace", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_iban_remove, "@iban:remove", "DE89370400440532013000", @r###" + { + "@iban:remove|DE89370400440532013000": { + "type": "string", + "value": "" + }, + "_meta": { + "@iban:remove|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:remove", + "x", + 0, + 0 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_iban_mask, "@iban:mask", "DE89370400440532013000", @r###" + { + "@iban:mask|DE89370400440532013000": { + "type": "string", + "value": "**********************" + }, + "_meta": { + "@iban:mask|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:mask", + "m", + 0, + 22 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + // MAC address rules + attribute_rule_test!(test_scrub_span_pii_string_rules_mac, "@mac", "4a:00:04:10:9b:50", @r###" + { + "@mac|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_mac_replace, "@mac:replace", "4a:00:04:10:9b:50", @r###" + { + "@mac:replace|4a:00:04:10:9b:50": { + "type": "string", + "value": "[mac]" + }, + "_meta": { + "@mac:replace|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:replace", + "s", + 0, + 5 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_mac_remove, "@mac:remove", "4a:00:04:10:9b:50", @r###" + { + "@mac:remove|4a:00:04:10:9b:50": { + "type": "string", + "value": "" + }, + "_meta": { + "@mac:remove|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:remove", + "x", + 0, + 0 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_mac_mask, "@mac:mask", "4a:00:04:10:9b:50", @r###" + { + "@mac:mask|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac:mask|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:mask", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + // UUID rules + attribute_rule_test!(test_scrub_span_pii_string_rules_uuid, "@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_replace, "@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "[uuid]" + }, + "_meta": { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:replace", + "s", + 0, + 6 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_remove, "@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "" + }, + "_meta": { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:remove", + "x", + 0, + 0 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_mask, "@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:mask", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + // IMEI rules + attribute_rule_test!(test_scrub_span_pii_string_rules_imei, "@imei", "356938035643809", @r###" + { + "@imei|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_imei_replace, "@imei:replace", "356938035643809", @r###" + { + "@imei:replace|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei:replace|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:replace", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_imei_remove, "@imei:remove", "356938035643809", @r###" + { + "@imei:remove|356938035643809": { + "type": "string", + "value": "" + }, + "_meta": { + "@imei:remove|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // PEM key rules + attribute_rule_test!( + test_scrub_span_pii_string_rules_pemkey, + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_span_pii_string_rules_pemkey_replace, + "@pemkey:replace", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:replace", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_span_pii_string_rules_pemkey_remove, + "@pemkey:remove", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:remove", + "x", + 31, + 31 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + // URL auth rules + attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth, "@urlauth", "https://username:password@example.com/", @r###" + { + "@urlauth|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_replace, "@urlauth:replace", "https://username:password@example.com/", @r###" + { + "@urlauth:replace|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth:replace|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:replace", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_remove, "@urlauth:remove", "https://username:password@example.com/", @r###" + { + "@urlauth:remove|https://username:password@example.com/": { + "type": "string", + "value": "https://@example.com/" + }, + "_meta": { + "@urlauth:remove|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:remove", + "x", + 8, + 8 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_mask, "@urlauth:mask", "https://username:password@example.com/", @r###" + { + "@urlauth:mask|https://username:password@example.com/": { + "type": "string", + "value": "https://*****************@example.com/" + }, + "_meta": { + "@urlauth:mask|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:mask", + "m", + 8, + 25 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + // US SSN rules + attribute_rule_test!(test_scrub_span_pii_string_rules_usssn, "@usssn", "078-05-1120", @r###" + { + "@usssn|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_replace, "@usssn:replace", "078-05-1120", @r###" + { + "@usssn:replace|078-05-1120": { + "type": "string", + "value": "[us-ssn]" + }, + "_meta": { + "@usssn:replace|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:replace", + "s", + 0, + 8 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_remove, "@usssn:remove", "078-05-1120", @r###" + { + "@usssn:remove|078-05-1120": { + "type": "string", + "value": "" + }, + "_meta": { + "@usssn:remove|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:remove", + "x", + 0, + 0 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_mask, "@usssn:mask", "078-05-1120", @r###" + { + "@usssn:mask|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn:mask|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:mask", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + // User path rules + attribute_rule_test!(test_scrub_span_pii_string_rules_userpath, "@userpath", "/Users/john/Documents", @r###" + { + "@userpath|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_replace, "@userpath:replace", "/Users/john/Documents", @r###" + { + "@userpath:replace|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath:replace|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:replace", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_remove, "@userpath:remove", "/Users/john/Documents", @r###" + { + "@userpath:remove|/Users/john/Documents": { + "type": "string", + "value": "/Users//Documents" + }, + "_meta": { + "@userpath:remove|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:remove", + "x", + 7, + 7 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_mask, "@userpath:mask", "/Users/john/Documents", @r###" + { + "@userpath:mask|/Users/john/Documents": { + "type": "string", + "value": "/Users/****/Documents" + }, + "_meta": { + "@userpath:mask|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:mask", + "m", + 7, + 11 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + // Password rules + attribute_rule_test!(test_scrub_span_pii_string_rules_password, "@password", "my_password_123", @r###" + { + "@password|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); // @password defaults to remove + attribute_rule_test!(test_scrub_span_pii_string_rules_password_remove, "@password:remove", "my_password_123", @r###" + { + "@password:remove|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password:remove|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_password_replace, "@password:replace", "my_password_123", @r###" + { + "@password:replace|my_password_123": { + "type": "string", + "value": "[password]" + }, + "_meta": { + "@password:replace|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:replace", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_span_pii_string_rules_password_mask, "@password:mask", "my_password_123", @r###" + { + "@password:mask|my_password_123": { + "type": "string", + "value": "***************" + }, + "_meta": { + "@password:mask|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:mask", + "m", + 0, + 15 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // Bearer token rules + attribute_rule_test!(test_scrub_span_pii_string_rules_bearer, "@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", @r###" + { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_span_pii_string_rules_bearer_replace, + "@bearer:replace", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:replace", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_span_pii_string_rules_bearer_remove, + "@bearer:remove", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "" + }, + "_meta": { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:remove", + "x", + 0, + 0 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_span_pii_string_rules_bearer_mask, + "@bearer:mask", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "*******************************************" + }, + "_meta": { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:mask", + "m", + 0, + 43 + ] + ], + "len": 43 + } + } } + } } + "###); #[test] fn test_scrub_span_implicit_attribute_value_does_not_match() { diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap deleted file mode 100644 index 5e0644e8858..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-10.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:replace|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard:replace|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:replace", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap deleted file mode 100644 index d6fcf4eeed4..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-11.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:remove|4242424242424242": { - "type": "string", - "value": "" - }, - "_meta": { - "@creditcard:remove|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap deleted file mode 100644 index 83b5941e72b..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-12.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:mask|4242424242424242": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@creditcard:mask|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap deleted file mode 100644 index 0098214af88..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-13.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap deleted file mode 100644 index 8e7180f1fb8..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-14.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:replace|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban:replace|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:replace", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap deleted file mode 100644 index e4e4f157cc7..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-15.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:remove|DE89370400440532013000": { - "type": "string", - "value": "" - }, - "_meta": { - "@iban:remove|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:remove", - "x", - 0, - 0 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap deleted file mode 100644 index 6e9e9a41591..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-16.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:mask|DE89370400440532013000": { - "type": "string", - "value": "**********************" - }, - "_meta": { - "@iban:mask|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:mask", - "m", - 0, - 22 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap deleted file mode 100644 index 1bb20ef1177..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-17.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap deleted file mode 100644 index 2252f0e360b..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-18.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:replace|4a:00:04:10:9b:50": { - "type": "string", - "value": "[mac]" - }, - "_meta": { - "@mac:replace|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:replace", - "s", - 0, - 5 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap deleted file mode 100644 index b8eae592a5e..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-19.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:remove|4a:00:04:10:9b:50": { - "type": "string", - "value": "" - }, - "_meta": { - "@mac:remove|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:remove", - "x", - 0, - 0 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap deleted file mode 100644 index 511c45134a8..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-2.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:replace|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip:replace|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap deleted file mode 100644 index eb85d44e55e..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-20.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:mask|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac:mask|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:mask", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap deleted file mode 100644 index 3985d61bac0..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-21.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap deleted file mode 100644 index 0e462295fdf..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-22.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "[uuid]" - }, - "_meta": { - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:replace", - "s", - 0, - 6 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap deleted file mode 100644 index bd65b71cc17..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-23.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "" - }, - "_meta": { - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:remove", - "x", - 0, - 0 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap deleted file mode 100644 index e9e60427027..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-24.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:mask", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap deleted file mode 100644 index 56b94c00632..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-25.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap deleted file mode 100644 index 54d8ffc74ea..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-26.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:replace|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei:replace|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:replace", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap deleted file mode 100644 index b2082a9a4ca..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-27.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:remove|356938035643809": { - "type": "string", - "value": "" - }, - "_meta": { - "@imei:remove|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap deleted file mode 100644 index 01619327b8e..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-28.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap deleted file mode 100644 index 1bc4eb0e266..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-29.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:replace", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap deleted file mode 100644 index 0af041ca2d7..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-3.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:remove|127.0.0.1": { - "type": "string", - "value": "" - }, - "_meta": { - "@ip:remove|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:remove", - "x", - 0, - 0 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap deleted file mode 100644 index 6ffe7daf5bb..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-30.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:remove", - "x", - 31, - 31 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap deleted file mode 100644 index 680380ffe6d..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-31.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap deleted file mode 100644 index e7ef4496618..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-32.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:replace|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth:replace|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:replace", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap deleted file mode 100644 index bb838483d5f..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-33.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:remove|https://username:password@example.com/": { - "type": "string", - "value": "https://@example.com/" - }, - "_meta": { - "@urlauth:remove|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:remove", - "x", - 8, - 8 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap deleted file mode 100644 index 19221198ab7..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-34.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:mask|https://username:password@example.com/": { - "type": "string", - "value": "https://*****************@example.com/" - }, - "_meta": { - "@urlauth:mask|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:mask", - "m", - 8, - 25 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap deleted file mode 100644 index a95c6a9d5fa..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-35.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap deleted file mode 100644 index f0b410d8b0c..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-36.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:replace|078-05-1120": { - "type": "string", - "value": "[us-ssn]" - }, - "_meta": { - "@usssn:replace|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:replace", - "s", - 0, - 8 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap deleted file mode 100644 index 823aa122201..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-37.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:remove|078-05-1120": { - "type": "string", - "value": "" - }, - "_meta": { - "@usssn:remove|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:remove", - "x", - 0, - 0 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap deleted file mode 100644 index a7713e60da8..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-38.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:mask|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn:mask|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:mask", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap deleted file mode 100644 index 726bafeb0b2..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-39.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap deleted file mode 100644 index 188963150d0..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-4.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:mask|127.0.0.1": { - "type": "string", - "value": "*********" - }, - "_meta": { - "@ip:mask|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:mask", - "m", - 0, - 9 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap deleted file mode 100644 index 22b8635d29b..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-40.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:replace|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath:replace|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:replace", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap deleted file mode 100644 index c8b4e7eb704..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-41.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:remove|/Users/john/Documents": { - "type": "string", - "value": "/Users//Documents" - }, - "_meta": { - "@userpath:remove|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:remove", - "x", - 7, - 7 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap deleted file mode 100644 index 22717e2fd8c..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-42.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:mask|/Users/john/Documents": { - "type": "string", - "value": "/Users/****/Documents" - }, - "_meta": { - "@userpath:mask|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:mask", - "m", - 7, - 11 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap deleted file mode 100644 index a09ff5b5ccd..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-43.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap deleted file mode 100644 index c9acdca4b62..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-44.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:remove|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password:remove|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap deleted file mode 100644 index 5c9917bbd01..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-45.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:replace|my_password_123": { - "type": "string", - "value": "[password]" - }, - "_meta": { - "@password:replace|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:replace", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap deleted file mode 100644 index 277805125cc..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-46.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:mask|my_password_123": { - "type": "string", - "value": "***************" - }, - "_meta": { - "@password:mask|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:mask", - "m", - 0, - 15 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap deleted file mode 100644 index 67f8b7f6852..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-47.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap deleted file mode 100644 index 832eae1a17f..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-48.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:replace", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap deleted file mode 100644 index 4fe5634c249..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-49.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "" - }, - "_meta": { - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:remove", - "x", - 0, - 0 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap deleted file mode 100644 index 5fbb49273d1..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-5.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap deleted file mode 100644 index f66f31b45b6..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-50.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "*******************************************" - }, - "_meta": { - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:mask", - "m", - 0, - 43 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap deleted file mode 100644 index 1e09eda548a..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-6.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:replace|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email:replace|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:replace", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap deleted file mode 100644 index bc35eb53873..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-7.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:remove|test@example.com": { - "type": "string", - "value": "" - }, - "_meta": { - "@email:remove|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap deleted file mode 100644 index 75dd60d6717..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-8.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:mask|test@example.com": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@email:mask|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap deleted file mode 100644 index 5f2fe545179..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules-9.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap b/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap deleted file mode 100644 index 67d74a47d45..00000000000 --- a/relay-server/src/processing/spans/snapshots/relay_server__processing__spans__process__tests__scrub_span_pii_string_rules.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/spans/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} From 6f8533fb6bee8dc094a39e3da6783402acb8eff9 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 14 Oct 2025 13:36:31 +0200 Subject: [PATCH 08/20] Factor out scrub_eap_item function --- relay-pii/src/eap.rs | 31 ++++++++++++++++++++ relay-pii/src/lib.rs | 2 ++ relay-server/src/processing/logs/process.rs | 22 ++++---------- relay-server/src/processing/spans/process.rs | 22 ++++---------- 4 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 relay-pii/src/eap.rs diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs new file mode 100644 index 00000000000..f8314a5b822 --- /dev/null +++ b/relay-pii/src/eap.rs @@ -0,0 +1,31 @@ +use relay_event_schema::processor::{ + ProcessValue, ProcessingResult, ProcessingState, ValueType, process_value, +}; +use relay_protocol::Annotated; + +use crate::{AttributeMode, PiiConfig, PiiProcessor}; + +pub fn scrub_eap_item( + value_type: ValueType, + item: &mut Annotated, + pii_config: Option<&PiiConfig>, + pii_config_from_scrubbing: Option<&PiiConfig>, +) -> ProcessingResult { + let state = ProcessingState::root().enter_borrowed("", None, [value_type]); + + if let Some(config) = pii_config { + let mut processor = PiiProcessor::new(config.compiled()) + // For advanced rules we want to treat attributes as objects. + .attribute_mode(AttributeMode::Object); + process_value(item, &mut processor, &state)?; + } + + if let Some(config) = pii_config_from_scrubbing { + let mut processor = PiiProcessor::new(config.compiled()) + // For "legacy" rules we want to identify attributes with their values. + .attribute_mode(AttributeMode::ValueOnly); + process_value(item, &mut processor, &state)?; + } + + Ok(()) +} diff --git a/relay-pii/src/lib.rs b/relay-pii/src/lib.rs index 4d1f108c26d..d4d5e52c9c5 100644 --- a/relay-pii/src/lib.rs +++ b/relay-pii/src/lib.rs @@ -11,6 +11,7 @@ mod builtin; mod compiledconfig; mod config; mod convert; +mod eap; mod generate_selectors; mod json; mod legacy; @@ -26,6 +27,7 @@ pub mod transform; pub use self::attachments::*; pub use self::compiledconfig::*; pub use self::config::*; +pub use self::eap::*; pub use self::generate_selectors::selector_suggestions_from_value; pub use self::json::*; pub use self::legacy::*; diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 5951db94978..19bc1e716eb 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -1,7 +1,6 @@ use relay_event_normalization::{SchemaProcessor, eap}; use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::{OurLog, OurLogHeader}; -use relay_pii::{AttributeMode, PiiProcessor}; use relay_protocol::Annotated; use relay_quotas::DataCategory; @@ -98,21 +97,12 @@ fn scrub_log(log: &mut Annotated, ctx: Context<'_>) -> Result<()> { .pii_config() .map_err(|e| Error::PiiConfig(e.clone()))?; - let state = ProcessingState::root().enter_borrowed("", None, [ValueType::OurLog]); - - if let Some(ref config) = ctx.project_info.config.pii_config { - let mut processor = PiiProcessor::new(config.compiled()) - // For advanced rules we want to treat attributes as objects. - .attribute_mode(AttributeMode::Object); - process_value(log, &mut processor, &state)?; - } - - if let Some(config) = pii_config_from_scrubbing { - let mut processor = PiiProcessor::new(config.compiled()) - // For "legacy" rules we want to identify attributes with their values. - .attribute_mode(AttributeMode::ValueOnly); - process_value(log, &mut processor, &state)?; - } + relay_pii::scrub_eap_item( + ValueType::OurLog, + log, + ctx.project_info.config.pii_config.as_ref(), + pii_config_from_scrubbing.as_ref(), + )?; Ok(()) } diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 412747045d1..807ebc0d31c 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -3,7 +3,6 @@ use relay_event_normalization::{ }; use relay_event_schema::processor::{ProcessingState, ValueType, process_value}; use relay_event_schema::protocol::SpanV2; -use relay_pii::{AttributeMode, PiiProcessor}; use relay_protocol::Annotated; use crate::envelope::{ContainerItems, Item, ItemContainer}; @@ -108,21 +107,12 @@ fn scrub_span(span: &mut Annotated, ctx: Context<'_>) -> Result<()> { .pii_config() .map_err(|e| Error::PiiConfig(e.clone()))?; - let state = ProcessingState::root().enter_borrowed("", None, [ValueType::Span]); - - if let Some(ref config) = ctx.project_info.config.pii_config { - let mut processor = PiiProcessor::new(config.compiled()) - // For advanced rules we want to treat attributes as objects. - .attribute_mode(AttributeMode::Object); - process_value(span, &mut processor, &state)?; - } - - if let Some(config) = pii_config_from_scrubbing { - let mut processor = PiiProcessor::new(config.compiled()) - // For "legacy" rules we want to identify attributes with their values. - .attribute_mode(AttributeMode::ValueOnly); - process_value(span, &mut processor, &state)?; - } + relay_pii::scrub_eap_item( + ValueType::Span, + span, + ctx.project_info.config.pii_config.as_ref(), + pii_config_from_scrubbing.as_ref(), + )?; Ok(()) } From c76fa43c8d67904c00e60834afbf75fc1197b982 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 14 Oct 2025 13:55:20 +0200 Subject: [PATCH 09/20] Move some span/log tests to relay-pii --- relay-pii/src/eap.rs | 1420 +++++++++++++++++ relay-server/src/processing/logs/process.rs | 130 -- ..._tests__scrub_log_pii_string_rules-10.snap | 27 - ..._tests__scrub_log_pii_string_rules-11.snap | 27 - ..._tests__scrub_log_pii_string_rules-12.snap | 27 - ..._tests__scrub_log_pii_string_rules-13.snap | 27 - ..._tests__scrub_log_pii_string_rules-14.snap | 27 - ..._tests__scrub_log_pii_string_rules-15.snap | 27 - ..._tests__scrub_log_pii_string_rules-16.snap | 27 - ..._tests__scrub_log_pii_string_rules-17.snap | 27 - ..._tests__scrub_log_pii_string_rules-18.snap | 27 - ..._tests__scrub_log_pii_string_rules-19.snap | 27 - ...__tests__scrub_log_pii_string_rules-2.snap | 27 - ..._tests__scrub_log_pii_string_rules-20.snap | 27 - ..._tests__scrub_log_pii_string_rules-21.snap | 27 - ..._tests__scrub_log_pii_string_rules-22.snap | 27 - ..._tests__scrub_log_pii_string_rules-23.snap | 27 - ..._tests__scrub_log_pii_string_rules-24.snap | 27 - ..._tests__scrub_log_pii_string_rules-25.snap | 27 - ..._tests__scrub_log_pii_string_rules-26.snap | 27 - ..._tests__scrub_log_pii_string_rules-27.snap | 27 - ..._tests__scrub_log_pii_string_rules-28.snap | 27 - ..._tests__scrub_log_pii_string_rules-29.snap | 27 - ...__tests__scrub_log_pii_string_rules-3.snap | 27 - ..._tests__scrub_log_pii_string_rules-30.snap | 27 - ..._tests__scrub_log_pii_string_rules-31.snap | 27 - ..._tests__scrub_log_pii_string_rules-32.snap | 27 - ..._tests__scrub_log_pii_string_rules-33.snap | 27 - ..._tests__scrub_log_pii_string_rules-34.snap | 27 - ..._tests__scrub_log_pii_string_rules-35.snap | 27 - ..._tests__scrub_log_pii_string_rules-36.snap | 27 - ..._tests__scrub_log_pii_string_rules-37.snap | 27 - ..._tests__scrub_log_pii_string_rules-38.snap | 27 - ..._tests__scrub_log_pii_string_rules-39.snap | 27 - ...__tests__scrub_log_pii_string_rules-4.snap | 27 - ..._tests__scrub_log_pii_string_rules-40.snap | 27 - ..._tests__scrub_log_pii_string_rules-41.snap | 27 - ..._tests__scrub_log_pii_string_rules-42.snap | 27 - ..._tests__scrub_log_pii_string_rules-43.snap | 27 - ..._tests__scrub_log_pii_string_rules-44.snap | 27 - ..._tests__scrub_log_pii_string_rules-45.snap | 27 - ..._tests__scrub_log_pii_string_rules-46.snap | 27 - ..._tests__scrub_log_pii_string_rules-47.snap | 27 - ..._tests__scrub_log_pii_string_rules-48.snap | 27 - ..._tests__scrub_log_pii_string_rules-49.snap | 27 - ...__tests__scrub_log_pii_string_rules-5.snap | 27 - ..._tests__scrub_log_pii_string_rules-50.snap | 27 - ...__tests__scrub_log_pii_string_rules-6.snap | 27 - ...__tests__scrub_log_pii_string_rules-7.snap | 27 - ...__tests__scrub_log_pii_string_rules-8.snap | 27 - ...__tests__scrub_log_pii_string_rules-9.snap | 27 - ...ss__tests__scrub_log_pii_string_rules.snap | 27 - relay-server/src/processing/spans/process.rs | 1383 ---------------- 53 files changed, 1420 insertions(+), 2863 deletions(-) delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap delete mode 100644 relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index f8314a5b822..393c0d34608 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -29,3 +29,1423 @@ pub fn scrub_eap_item( Ok(()) } + +#[cfg(test)] +mod tests { + use relay_event_schema::processor::ValueType; + use relay_event_schema::protocol::{OurLog, SpanV2}; + use relay_protocol::{Annotated, SerializableAnnotated}; + + use crate::PiiConfig; + + macro_rules! attribute_rule_test { + ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => { + #[test] + fn $test_name() { + let config = serde_json::from_value::(serde_json::json!({ + "applications": { + "$string": [$rule_type] + } + })) + .unwrap(); + + let mut scrubbing_config = crate::DataScrubbingConfig::default(); + scrubbing_config.scrub_data = true; + scrubbing_config.scrub_defaults = false; + scrubbing_config.scrub_ip_addresses = false; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + let span_json = format!(r#"{{ + "start_timestamp": 1544719859.0, + "end_timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "test", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = $rule_type, + value = $test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut span = Annotated::::from_json(&span_json).unwrap(); + + crate::scrub_eap_item(ValueType::Span, &mut span, Some(&config), scrubbing_config.as_ref()).unwrap(); + + insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&span.value().unwrap().attributes), @$snapshot)); + + let log_json = format!(r#"{{ + "timestamp": 1544719860.0, + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "level": "info", + "body": "Test log", + "attributes": {{ + "{rule_type}|{value}": {{ + "type": "string", + "value": "{value}" + }} + }} + }}"#, + rule_type = $rule_type, + value = $test_value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace('\n', "\\n") + ); + + let mut log = Annotated::::from_json(&log_json).unwrap(); + + crate::scrub_eap_item(ValueType::OurLog, &mut log, Some(&config), scrubbing_config.as_ref()).unwrap(); + + insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&log.value().unwrap().attributes), @$snapshot)); + } + }; + } + + // IP rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip, "@ip", "127.0.0.1", @r###" + { + "@ip|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_replace, "@ip:replace", "127.0.0.1", @r###" + { + "@ip:replace|127.0.0.1": { + "type": "string", + "value": "[ip]" + }, + "_meta": { + "@ip:replace|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_remove, "@ip:remove", "127.0.0.1", @r###" + { + "@ip:remove|127.0.0.1": { + "type": "string", + "value": "" + }, + "_meta": { + "@ip:remove|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:remove", + "x", + 0, + 0 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_mask, "@ip:mask", "127.0.0.1", @r###" + { + "@ip:mask|127.0.0.1": { + "type": "string", + "value": "*********" + }, + "_meta": { + "@ip:mask|127.0.0.1": { + "value": { + "": { + "rem": [ + [ + "@ip:mask", + "m", + 0, + 9 + ] + ], + "len": 9 + } + } + } + } + } + "###); + + // Email rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email, "@email", "test@example.com", @r###" + { + "@email|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_replace, "@email:replace", "test@example.com", @r###" + { + "@email:replace|test@example.com": { + "type": "string", + "value": "[email]" + }, + "_meta": { + "@email:replace|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:replace", + "s", + 0, + 7 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_remove, "@email:remove", "test@example.com", @r###" + { + "@email:remove|test@example.com": { + "type": "string", + "value": "" + }, + "_meta": { + "@email:remove|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_mask, "@email:mask", "test@example.com", @r###" + { + "@email:mask|test@example.com": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@email:mask|test@example.com": { + "value": { + "": { + "rem": [ + [ + "@email:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // Credit card rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard, "@creditcard", "4242424242424242", @r###" + { + "@creditcard|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_replace, "@creditcard:replace", "4242424242424242", @r###" + { + "@creditcard:replace|4242424242424242": { + "type": "string", + "value": "[creditcard]" + }, + "_meta": { + "@creditcard:replace|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:replace", + "s", + 0, + 12 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_remove, "@creditcard:remove", "4242424242424242", @r###" + { + "@creditcard:remove|4242424242424242": { + "type": "string", + "value": "" + }, + "_meta": { + "@creditcard:remove|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:remove", + "x", + 0, + 0 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_mask, "@creditcard:mask", "4242424242424242", @r###" + { + "@creditcard:mask|4242424242424242": { + "type": "string", + "value": "****************" + }, + "_meta": { + "@creditcard:mask|4242424242424242": { + "value": { + "": { + "rem": [ + [ + "@creditcard:mask", + "m", + 0, + 16 + ] + ], + "len": 16 + } + } + } + } + } + "###); + + // IBAN rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban, "@iban", "DE89370400440532013000", @r###" + { + "@iban|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_replace, "@iban:replace", "DE89370400440532013000", @r###" + { + "@iban:replace|DE89370400440532013000": { + "type": "string", + "value": "[iban]" + }, + "_meta": { + "@iban:replace|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:replace", + "s", + 0, + 6 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_remove, "@iban:remove", "DE89370400440532013000", @r###" + { + "@iban:remove|DE89370400440532013000": { + "type": "string", + "value": "" + }, + "_meta": { + "@iban:remove|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:remove", + "x", + 0, + 0 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_mask, "@iban:mask", "DE89370400440532013000", @r###" + { + "@iban:mask|DE89370400440532013000": { + "type": "string", + "value": "**********************" + }, + "_meta": { + "@iban:mask|DE89370400440532013000": { + "value": { + "": { + "rem": [ + [ + "@iban:mask", + "m", + 0, + 22 + ] + ], + "len": 22 + } + } + } + } + } + "###); + + // MAC address rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac, "@mac", "4a:00:04:10:9b:50", @r###" + { + "@mac|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_replace, "@mac:replace", "4a:00:04:10:9b:50", @r###" + { + "@mac:replace|4a:00:04:10:9b:50": { + "type": "string", + "value": "[mac]" + }, + "_meta": { + "@mac:replace|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:replace", + "s", + 0, + 5 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_remove, "@mac:remove", "4a:00:04:10:9b:50", @r###" + { + "@mac:remove|4a:00:04:10:9b:50": { + "type": "string", + "value": "" + }, + "_meta": { + "@mac:remove|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:remove", + "x", + 0, + 0 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_mask, "@mac:mask", "4a:00:04:10:9b:50", @r###" + { + "@mac:mask|4a:00:04:10:9b:50": { + "type": "string", + "value": "*****************" + }, + "_meta": { + "@mac:mask|4a:00:04:10:9b:50": { + "value": { + "": { + "rem": [ + [ + "@mac:mask", + "m", + 0, + 17 + ] + ], + "len": 17 + } + } + } + } + } + "###); + + // UUID rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid, "@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_replace, "@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "[uuid]" + }, + "_meta": { + "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:replace", + "s", + 0, + 6 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_remove, "@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "" + }, + "_meta": { + "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:remove", + "x", + 0, + 0 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_mask, "@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" + { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "type": "string", + "value": "************************************" + }, + "_meta": { + "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { + "value": { + "": { + "rem": [ + [ + "@uuid:mask", + "m", + 0, + 36 + ] + ], + "len": 36 + } + } + } + } + } + "###); + + // IMEI rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei, "@imei", "356938035643809", @r###" + { + "@imei|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_replace, "@imei:replace", "356938035643809", @r###" + { + "@imei:replace|356938035643809": { + "type": "string", + "value": "[imei]" + }, + "_meta": { + "@imei:replace|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:replace", + "s", + 0, + 6 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_remove, "@imei:remove", "356938035643809", @r###" + { + "@imei:remove|356938035643809": { + "type": "string", + "value": "" + }, + "_meta": { + "@imei:remove|356938035643809": { + "value": { + "": { + "rem": [ + [ + "@imei:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // PEM key rules + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey, + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey_replace, + "@pemkey:replace", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:replace", + "s", + 31, + 39 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_pemkey_remove, + "@pemkey:remove", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + @r###" + { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "type": "string", + "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" + }, + "_meta": { + "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { + "value": { + "": { + "rem": [ + [ + "@pemkey:remove", + "x", + 31, + 31 + ] + ], + "len": 124 + } + } + } + } + } + "###); + + // URL auth rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth, "@urlauth", "https://username:password@example.com/", @r###" + { + "@urlauth|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_replace, "@urlauth:replace", "https://username:password@example.com/", @r###" + { + "@urlauth:replace|https://username:password@example.com/": { + "type": "string", + "value": "https://[auth]@example.com/" + }, + "_meta": { + "@urlauth:replace|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:replace", + "s", + 8, + 14 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_remove, "@urlauth:remove", "https://username:password@example.com/", @r###" + { + "@urlauth:remove|https://username:password@example.com/": { + "type": "string", + "value": "https://@example.com/" + }, + "_meta": { + "@urlauth:remove|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:remove", + "x", + 8, + 8 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_mask, "@urlauth:mask", "https://username:password@example.com/", @r###" + { + "@urlauth:mask|https://username:password@example.com/": { + "type": "string", + "value": "https://*****************@example.com/" + }, + "_meta": { + "@urlauth:mask|https://username:password@example.com/": { + "value": { + "": { + "rem": [ + [ + "@urlauth:mask", + "m", + 8, + 25 + ] + ], + "len": 38 + } + } + } + } + } + "###); + + // US SSN rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn, "@usssn", "078-05-1120", @r###" + { + "@usssn|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_replace, "@usssn:replace", "078-05-1120", @r###" + { + "@usssn:replace|078-05-1120": { + "type": "string", + "value": "[us-ssn]" + }, + "_meta": { + "@usssn:replace|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:replace", + "s", + 0, + 8 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_remove, "@usssn:remove", "078-05-1120", @r###" + { + "@usssn:remove|078-05-1120": { + "type": "string", + "value": "" + }, + "_meta": { + "@usssn:remove|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:remove", + "x", + 0, + 0 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_mask, "@usssn:mask", "078-05-1120", @r###" + { + "@usssn:mask|078-05-1120": { + "type": "string", + "value": "***********" + }, + "_meta": { + "@usssn:mask|078-05-1120": { + "value": { + "": { + "rem": [ + [ + "@usssn:mask", + "m", + 0, + 11 + ] + ], + "len": 11 + } + } + } + } + } + "###); + + // User path rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath, "@userpath", "/Users/john/Documents", @r###" + { + "@userpath|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_replace, "@userpath:replace", "/Users/john/Documents", @r###" + { + "@userpath:replace|/Users/john/Documents": { + "type": "string", + "value": "/Users/[user]/Documents" + }, + "_meta": { + "@userpath:replace|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:replace", + "s", + 7, + 13 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_remove, "@userpath:remove", "/Users/john/Documents", @r###" + { + "@userpath:remove|/Users/john/Documents": { + "type": "string", + "value": "/Users//Documents" + }, + "_meta": { + "@userpath:remove|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:remove", + "x", + 7, + 7 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_mask, "@userpath:mask", "/Users/john/Documents", @r###" + { + "@userpath:mask|/Users/john/Documents": { + "type": "string", + "value": "/Users/****/Documents" + }, + "_meta": { + "@userpath:mask|/Users/john/Documents": { + "value": { + "": { + "rem": [ + [ + "@userpath:mask", + "m", + 7, + 11 + ] + ], + "len": 21 + } + } + } + } + } + "###); + + // Password rules + // @password defaults to remove + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password, "@password", "my_password_123", @r###" + { + "@password|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_remove, "@password:remove", "my_password_123", @r###" + { + "@password:remove|my_password_123": { + "type": "string", + "value": "" + }, + "_meta": { + "@password:remove|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:remove", + "x", + 0, + 0 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_replace, "@password:replace", "my_password_123", @r###" + { + "@password:replace|my_password_123": { + "type": "string", + "value": "[password]" + }, + "_meta": { + "@password:replace|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:replace", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_mask, "@password:mask", "my_password_123", @r###" + { + "@password:mask|my_password_123": { + "type": "string", + "value": "***************" + }, + "_meta": { + "@password:mask|my_password_123": { + "value": { + "": { + "rem": [ + [ + "@password:mask", + "m", + 0, + 15 + ] + ], + "len": 15 + } + } + } + } + } + "###); + + // Bearer token rules + attribute_rule_test!(test_scrub_attributes_pii_string_rules_bearer, "@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", @r###" + { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_replace, + "@bearer:replace", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "Bearer [token]" + }, + "_meta": { + "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:replace", + "s", + 0, + 14 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_remove, + "@bearer:remove", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "" + }, + "_meta": { + "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:remove", + "x", + 0, + 0 + ] + ], + "len": 43 + } + } + } + } + } + "###); + + attribute_rule_test!( + test_scrub_attributes_pii_string_rules_bearer_mask, + "@bearer:mask", + "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + @r###" + { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "type": "string", + "value": "*******************************************" + }, + "_meta": { + "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { + "value": { + "": { + "rem": [ + [ + "@bearer:mask", + "m", + 0, + 43 + ] + ], + "len": 43 + } + } + } + } + } + "###); +} diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 19bc1e716eb..32e6ebeaeb2 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -929,136 +929,6 @@ mod tests { "###); } - #[test] - fn test_scrub_log_pii_string_rules() { - let test_cases = vec![ - // IP rules - ("@ip", "127.0.0.1"), - ("@ip:replace", "127.0.0.1"), - ("@ip:remove", "127.0.0.1"), - ("@ip:mask", "127.0.0.1"), - // Email rules - ("@email", "test@example.com"), - ("@email:replace", "test@example.com"), - ("@email:remove", "test@example.com"), - ("@email:mask", "test@example.com"), - // Credit card rules - ("@creditcard", "4242424242424242"), - ("@creditcard:replace", "4242424242424242"), - ("@creditcard:remove", "4242424242424242"), - ("@creditcard:mask", "4242424242424242"), - // IBAN rules - ("@iban", "DE89370400440532013000"), - ("@iban:replace", "DE89370400440532013000"), - ("@iban:remove", "DE89370400440532013000"), - ("@iban:mask", "DE89370400440532013000"), - // MAC address rules - ("@mac", "4a:00:04:10:9b:50"), - ("@mac:replace", "4a:00:04:10:9b:50"), - ("@mac:remove", "4a:00:04:10:9b:50"), - ("@mac:mask", "4a:00:04:10:9b:50"), - // UUID rules - ("@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - ("@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1"), - // IMEI rules - ("@imei", "356938035643809"), - ("@imei:replace", "356938035643809"), - ("@imei:remove", "356938035643809"), - // PEM key rules - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:replace", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - ( - "@pemkey:remove", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - ), - // URL auth rules - ("@urlauth", "https://username:password@example.com/"), - ("@urlauth:replace", "https://username:password@example.com/"), - ("@urlauth:remove", "https://username:password@example.com/"), - ("@urlauth:mask", "https://username:password@example.com/"), - // US SSN rules - ("@usssn", "078-05-1120"), - ("@usssn:replace", "078-05-1120"), - ("@usssn:remove", "078-05-1120"), - ("@usssn:mask", "078-05-1120"), - // User path rules - ("@userpath", "/Users/john/Documents"), - ("@userpath:replace", "/Users/john/Documents"), - ("@userpath:remove", "/Users/john/Documents"), - ("@userpath:mask", "/Users/john/Documents"), - // Password rules - ("@password", "my_password_123"), // @password defaults to remove - ("@password:remove", "my_password_123"), - ("@password:replace", "my_password_123"), - ("@password:mask", "my_password_123"), - // Bearer token rules - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"), - ( - "@bearer:replace", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:remove", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ( - "@bearer:mask", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - ), - ]; - - for (rule_type, test_value) in test_cases { - let json = format!( - r#"{{ - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log", - "attributes": {{ - "{rule_type}|{value}": {{ - "type": "string", - "value": "{value}" - }} - }} - }}"#, - rule_type = rule_type, - value = test_value - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n") - ); - - let mut data = Annotated::::from_json(&json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "applications": { - "$string": [rule_type] - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes)); - } - } - #[test] fn test_scrub_log_base_fields() { let json = r#" diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap deleted file mode 100644 index 41c9d6f79ca..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-10.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:replace|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard:replace|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:replace", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap deleted file mode 100644 index 913c5f144b6..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-11.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:remove|4242424242424242": { - "type": "string", - "value": "" - }, - "_meta": { - "@creditcard:remove|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap deleted file mode 100644 index 657f4914f6a..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-12.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard:mask|4242424242424242": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@creditcard:mask|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap deleted file mode 100644 index 19a90e43cf7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-13.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap deleted file mode 100644 index 9fd82d71461..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-14.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:replace|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban:replace|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:replace", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap deleted file mode 100644 index ee757166106..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-15.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:remove|DE89370400440532013000": { - "type": "string", - "value": "" - }, - "_meta": { - "@iban:remove|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:remove", - "x", - 0, - 0 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap deleted file mode 100644 index 28fc20365c3..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-16.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@iban:mask|DE89370400440532013000": { - "type": "string", - "value": "**********************" - }, - "_meta": { - "@iban:mask|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:mask", - "m", - 0, - 22 - ] - ], - "len": 22 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap deleted file mode 100644 index a31b5045b9c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-17.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap deleted file mode 100644 index 374def85bb7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-18.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:replace|4a:00:04:10:9b:50": { - "type": "string", - "value": "[mac]" - }, - "_meta": { - "@mac:replace|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:replace", - "s", - 0, - 5 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap deleted file mode 100644 index 53d0822aacb..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-19.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:remove|4a:00:04:10:9b:50": { - "type": "string", - "value": "" - }, - "_meta": { - "@mac:remove|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:remove", - "x", - 0, - 0 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap deleted file mode 100644 index bfab00f6000..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-2.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:replace|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip:replace|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap deleted file mode 100644 index 0e54edf838d..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-20.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@mac:mask|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac:mask|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:mask", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap deleted file mode 100644 index 9d2fdf8e1b3..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-21.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap deleted file mode 100644 index 6add3fbe4c2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-22.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "[uuid]" - }, - "_meta": { - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:replace", - "s", - 0, - 6 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap deleted file mode 100644 index ecd17f3f6a2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-23.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "" - }, - "_meta": { - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:remove", - "x", - 0, - 0 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap deleted file mode 100644 index 9dd20e89202..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-24.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:mask", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap deleted file mode 100644 index 0091511248e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-25.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap deleted file mode 100644 index f05004683e7..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-26.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:replace|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei:replace|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:replace", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap deleted file mode 100644 index f207cc734da..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-27.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@imei:remove|356938035643809": { - "type": "string", - "value": "" - }, - "_meta": { - "@imei:remove|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap deleted file mode 100644 index a547b9b429e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-28.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap deleted file mode 100644 index 23a25e68a32..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-29.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:replace", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap deleted file mode 100644 index 8a66721fcb4..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-3.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:remove|127.0.0.1": { - "type": "string", - "value": "" - }, - "_meta": { - "@ip:remove|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:remove", - "x", - 0, - 0 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap deleted file mode 100644 index dae1df1dca2..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-30.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:remove", - "x", - 31, - 31 - ] - ], - "len": 124 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap deleted file mode 100644 index 47d54445c74..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-31.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap deleted file mode 100644 index f5167af0386..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-32.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:replace|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth:replace|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:replace", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap deleted file mode 100644 index 4d87f406b35..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-33.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:remove|https://username:password@example.com/": { - "type": "string", - "value": "https://@example.com/" - }, - "_meta": { - "@urlauth:remove|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:remove", - "x", - 8, - 8 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap deleted file mode 100644 index 4223b7551db..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-34.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@urlauth:mask|https://username:password@example.com/": { - "type": "string", - "value": "https://*****************@example.com/" - }, - "_meta": { - "@urlauth:mask|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:mask", - "m", - 8, - 25 - ] - ], - "len": 38 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap deleted file mode 100644 index 2f67315fcd1..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-35.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap deleted file mode 100644 index 5e97251cb03..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-36.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:replace|078-05-1120": { - "type": "string", - "value": "[us-ssn]" - }, - "_meta": { - "@usssn:replace|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:replace", - "s", - 0, - 8 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap deleted file mode 100644 index e3a7906f097..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-37.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:remove|078-05-1120": { - "type": "string", - "value": "" - }, - "_meta": { - "@usssn:remove|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:remove", - "x", - 0, - 0 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap deleted file mode 100644 index 9623a53cea0..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-38.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@usssn:mask|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn:mask|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:mask", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap deleted file mode 100644 index a66df61b449..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-39.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap deleted file mode 100644 index ce8078e830c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-4.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip:mask|127.0.0.1": { - "type": "string", - "value": "*********" - }, - "_meta": { - "@ip:mask|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:mask", - "m", - 0, - 9 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap deleted file mode 100644 index 547fd830f26..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-40.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:replace|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath:replace|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:replace", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap deleted file mode 100644 index 5b05bee508e..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-41.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:remove|/Users/john/Documents": { - "type": "string", - "value": "/Users//Documents" - }, - "_meta": { - "@userpath:remove|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:remove", - "x", - 7, - 7 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap deleted file mode 100644 index 08036842544..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-42.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@userpath:mask|/Users/john/Documents": { - "type": "string", - "value": "/Users/****/Documents" - }, - "_meta": { - "@userpath:mask|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:mask", - "m", - 7, - 11 - ] - ], - "len": 21 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap deleted file mode 100644 index 5747256ab36..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-43.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap deleted file mode 100644 index bfe78f0ce11..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-44.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:remove|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password:remove|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap deleted file mode 100644 index 1146e3722c8..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-45.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:replace|my_password_123": { - "type": "string", - "value": "[password]" - }, - "_meta": { - "@password:replace|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:replace", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap deleted file mode 100644 index c2534332bb1..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-46.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@password:mask|my_password_123": { - "type": "string", - "value": "***************" - }, - "_meta": { - "@password:mask|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:mask", - "m", - 0, - 15 - ] - ], - "len": 15 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap deleted file mode 100644 index f5cac4a7f8f..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-47.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap deleted file mode 100644 index 5d31471635c..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-48.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:replace", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap deleted file mode 100644 index d4c861f0031..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-49.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "" - }, - "_meta": { - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:remove", - "x", - 0, - 0 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap deleted file mode 100644 index fbb15d10c5a..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-5.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap deleted file mode 100644 index 498748ea4e4..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-50.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "*******************************************" - }, - "_meta": { - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:mask", - "m", - 0, - 43 - ] - ], - "len": 43 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap deleted file mode 100644 index 4f8eafb6159..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-6.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:replace|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email:replace|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:replace", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap deleted file mode 100644 index a52fd1dce91..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-7.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:remove|test@example.com": { - "type": "string", - "value": "" - }, - "_meta": { - "@email:remove|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap deleted file mode 100644 index 6e00d96e019..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-8.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@email:mask|test@example.com": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@email:mask|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap deleted file mode 100644 index d16ef12fee6..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules-9.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@creditcard|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } -} diff --git a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap b/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap deleted file mode 100644 index 370b5fc7527..00000000000 --- a/relay-server/src/processing/logs/snapshots/relay_server__processing__logs__process__tests__scrub_log_pii_string_rules.snap +++ /dev/null @@ -1,27 +0,0 @@ ---- -source: relay-server/src/processing/logs/process.rs -expression: SerializableAnnotated(&data.value().unwrap().attributes) ---- -{ - "@ip|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } -} diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 807ebc0d31c..ae5499cfea7 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -1371,1389 +1371,6 @@ mod tests { "###); } - macro_rules! attribute_rule_test { - ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => { - #[test] - fn $test_name() { - let json = format!(r#"{{ - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": {{ - "{rule_type}|{value}": {{ - "type": "string", - "value": "{value}" - }} - }} - }}"#, - rule_type = $rule_type, - value = $test_value - .replace('\\', "\\\\") - .replace('"', "\\\"") - .replace('\n', "\\n") - ); - - let mut data = Annotated::::from_json(&json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "applications": { - "$string": [$rule_type] - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @$snapshot); - } - }; - } - - // IP rules - attribute_rule_test!(test_scrub_span_pii_string_rules_ip, "@ip", "127.0.0.1", @r###" - { - "@ip|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_ip_replace, "@ip:replace", "127.0.0.1", @r###" - { - "@ip:replace|127.0.0.1": { - "type": "string", - "value": "[ip]" - }, - "_meta": { - "@ip:replace|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 9 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_ip_remove, "@ip:remove", "127.0.0.1", @r###" - { - "@ip:remove|127.0.0.1": { - "type": "string", - "value": "" - }, - "_meta": { - "@ip:remove|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:remove", - "x", - 0, - 0 - ] - ], - "len": 9 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_ip_mask, "@ip:mask", "127.0.0.1", @r###" - { - "@ip:mask|127.0.0.1": { - "type": "string", - "value": "*********" - }, - "_meta": { - "@ip:mask|127.0.0.1": { - "value": { - "": { - "rem": [ - [ - "@ip:mask", - "m", - 0, - 9 - ] - ], - "len": 9 - } - } - } - } - } - "###); - - // Email rules - attribute_rule_test!(test_scrub_span_pii_string_rules_email, "@email", "test@example.com", @r###" - { - "@email|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_email_replace, "@email:replace", "test@example.com", @r###" - { - "@email:replace|test@example.com": { - "type": "string", - "value": "[email]" - }, - "_meta": { - "@email:replace|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:replace", - "s", - 0, - 7 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_email_remove, "@email:remove", "test@example.com", @r###" - { - "@email:remove|test@example.com": { - "type": "string", - "value": "" - }, - "_meta": { - "@email:remove|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_email_mask, "@email:mask", "test@example.com", @r###" - { - "@email:mask|test@example.com": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@email:mask|test@example.com": { - "value": { - "": { - "rem": [ - [ - "@email:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - // Credit card rules - attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard, "@creditcard", "4242424242424242", @r###" - { - "@creditcard|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_replace, "@creditcard:replace", "4242424242424242", @r###" - { - "@creditcard:replace|4242424242424242": { - "type": "string", - "value": "[creditcard]" - }, - "_meta": { - "@creditcard:replace|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:replace", - "s", - 0, - 12 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_remove, "@creditcard:remove", "4242424242424242", @r###" - { - "@creditcard:remove|4242424242424242": { - "type": "string", - "value": "" - }, - "_meta": { - "@creditcard:remove|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:remove", - "x", - 0, - 0 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_creditcard_mask, "@creditcard:mask", "4242424242424242", @r###" - { - "@creditcard:mask|4242424242424242": { - "type": "string", - "value": "****************" - }, - "_meta": { - "@creditcard:mask|4242424242424242": { - "value": { - "": { - "rem": [ - [ - "@creditcard:mask", - "m", - 0, - 16 - ] - ], - "len": 16 - } - } - } - } - } - "###); - - // IBAN rules - attribute_rule_test!(test_scrub_span_pii_string_rules_iban, "@iban", "DE89370400440532013000", @r###" - { - "@iban|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_iban_replace, "@iban:replace", "DE89370400440532013000", @r###" - { - "@iban:replace|DE89370400440532013000": { - "type": "string", - "value": "[iban]" - }, - "_meta": { - "@iban:replace|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:replace", - "s", - 0, - 6 - ] - ], - "len": 22 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_iban_remove, "@iban:remove", "DE89370400440532013000", @r###" - { - "@iban:remove|DE89370400440532013000": { - "type": "string", - "value": "" - }, - "_meta": { - "@iban:remove|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:remove", - "x", - 0, - 0 - ] - ], - "len": 22 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_iban_mask, "@iban:mask", "DE89370400440532013000", @r###" - { - "@iban:mask|DE89370400440532013000": { - "type": "string", - "value": "**********************" - }, - "_meta": { - "@iban:mask|DE89370400440532013000": { - "value": { - "": { - "rem": [ - [ - "@iban:mask", - "m", - 0, - 22 - ] - ], - "len": 22 - } - } - } - } - } - "###); - - // MAC address rules - attribute_rule_test!(test_scrub_span_pii_string_rules_mac, "@mac", "4a:00:04:10:9b:50", @r###" - { - "@mac|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_mac_replace, "@mac:replace", "4a:00:04:10:9b:50", @r###" - { - "@mac:replace|4a:00:04:10:9b:50": { - "type": "string", - "value": "[mac]" - }, - "_meta": { - "@mac:replace|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:replace", - "s", - 0, - 5 - ] - ], - "len": 17 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_mac_remove, "@mac:remove", "4a:00:04:10:9b:50", @r###" - { - "@mac:remove|4a:00:04:10:9b:50": { - "type": "string", - "value": "" - }, - "_meta": { - "@mac:remove|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:remove", - "x", - 0, - 0 - ] - ], - "len": 17 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_mac_mask, "@mac:mask", "4a:00:04:10:9b:50", @r###" - { - "@mac:mask|4a:00:04:10:9b:50": { - "type": "string", - "value": "*****************" - }, - "_meta": { - "@mac:mask|4a:00:04:10:9b:50": { - "value": { - "": { - "rem": [ - [ - "@mac:mask", - "m", - 0, - 17 - ] - ], - "len": 17 - } - } - } - } - } - "###); - - // UUID rules - attribute_rule_test!(test_scrub_span_pii_string_rules_uuid, "@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" - { - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_replace, "@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" - { - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "[uuid]" - }, - "_meta": { - "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:replace", - "s", - 0, - 6 - ] - ], - "len": 36 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_remove, "@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" - { - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "" - }, - "_meta": { - "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:remove", - "x", - 0, - 0 - ] - ], - "len": 36 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_uuid_mask, "@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###" - { - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "type": "string", - "value": "************************************" - }, - "_meta": { - "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": { - "value": { - "": { - "rem": [ - [ - "@uuid:mask", - "m", - 0, - 36 - ] - ], - "len": 36 - } - } - } - } - } - "###); - - // IMEI rules - attribute_rule_test!(test_scrub_span_pii_string_rules_imei, "@imei", "356938035643809", @r###" - { - "@imei|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_imei_replace, "@imei:replace", "356938035643809", @r###" - { - "@imei:replace|356938035643809": { - "type": "string", - "value": "[imei]" - }, - "_meta": { - "@imei:replace|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:replace", - "s", - 0, - 6 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_imei_remove, "@imei:remove", "356938035643809", @r###" - { - "@imei:remove|356938035643809": { - "type": "string", - "value": "" - }, - "_meta": { - "@imei:remove|356938035643809": { - "value": { - "": { - "rem": [ - [ - "@imei:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - // PEM key rules - attribute_rule_test!( - test_scrub_span_pii_string_rules_pemkey, - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - @r###" - { - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } - } - "###); - - attribute_rule_test!( - test_scrub_span_pii_string_rules_pemkey_replace, - "@pemkey:replace", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - @r###" - { - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:replace", - "s", - 31, - 39 - ] - ], - "len": 124 - } - } - } - } - } - "###); - - attribute_rule_test!( - test_scrub_span_pii_string_rules_pemkey_remove, - "@pemkey:remove", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - @r###" - { - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "type": "string", - "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----" - }, - "_meta": { - "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": { - "value": { - "": { - "rem": [ - [ - "@pemkey:remove", - "x", - 31, - 31 - ] - ], - "len": 124 - } - } - } - } - } - "###); - - // URL auth rules - attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth, "@urlauth", "https://username:password@example.com/", @r###" - { - "@urlauth|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_replace, "@urlauth:replace", "https://username:password@example.com/", @r###" - { - "@urlauth:replace|https://username:password@example.com/": { - "type": "string", - "value": "https://[auth]@example.com/" - }, - "_meta": { - "@urlauth:replace|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:replace", - "s", - 8, - 14 - ] - ], - "len": 38 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_remove, "@urlauth:remove", "https://username:password@example.com/", @r###" - { - "@urlauth:remove|https://username:password@example.com/": { - "type": "string", - "value": "https://@example.com/" - }, - "_meta": { - "@urlauth:remove|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:remove", - "x", - 8, - 8 - ] - ], - "len": 38 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_urlauth_mask, "@urlauth:mask", "https://username:password@example.com/", @r###" - { - "@urlauth:mask|https://username:password@example.com/": { - "type": "string", - "value": "https://*****************@example.com/" - }, - "_meta": { - "@urlauth:mask|https://username:password@example.com/": { - "value": { - "": { - "rem": [ - [ - "@urlauth:mask", - "m", - 8, - 25 - ] - ], - "len": 38 - } - } - } - } - } - "###); - - // US SSN rules - attribute_rule_test!(test_scrub_span_pii_string_rules_usssn, "@usssn", "078-05-1120", @r###" - { - "@usssn|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_replace, "@usssn:replace", "078-05-1120", @r###" - { - "@usssn:replace|078-05-1120": { - "type": "string", - "value": "[us-ssn]" - }, - "_meta": { - "@usssn:replace|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:replace", - "s", - 0, - 8 - ] - ], - "len": 11 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_remove, "@usssn:remove", "078-05-1120", @r###" - { - "@usssn:remove|078-05-1120": { - "type": "string", - "value": "" - }, - "_meta": { - "@usssn:remove|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:remove", - "x", - 0, - 0 - ] - ], - "len": 11 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_usssn_mask, "@usssn:mask", "078-05-1120", @r###" - { - "@usssn:mask|078-05-1120": { - "type": "string", - "value": "***********" - }, - "_meta": { - "@usssn:mask|078-05-1120": { - "value": { - "": { - "rem": [ - [ - "@usssn:mask", - "m", - 0, - 11 - ] - ], - "len": 11 - } - } - } - } - } - "###); - - // User path rules - attribute_rule_test!(test_scrub_span_pii_string_rules_userpath, "@userpath", "/Users/john/Documents", @r###" - { - "@userpath|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_replace, "@userpath:replace", "/Users/john/Documents", @r###" - { - "@userpath:replace|/Users/john/Documents": { - "type": "string", - "value": "/Users/[user]/Documents" - }, - "_meta": { - "@userpath:replace|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:replace", - "s", - 7, - 13 - ] - ], - "len": 21 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_remove, "@userpath:remove", "/Users/john/Documents", @r###" - { - "@userpath:remove|/Users/john/Documents": { - "type": "string", - "value": "/Users//Documents" - }, - "_meta": { - "@userpath:remove|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:remove", - "x", - 7, - 7 - ] - ], - "len": 21 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_userpath_mask, "@userpath:mask", "/Users/john/Documents", @r###" - { - "@userpath:mask|/Users/john/Documents": { - "type": "string", - "value": "/Users/****/Documents" - }, - "_meta": { - "@userpath:mask|/Users/john/Documents": { - "value": { - "": { - "rem": [ - [ - "@userpath:mask", - "m", - 7, - 11 - ] - ], - "len": 21 - } - } - } - } - } - "###); - - // Password rules - attribute_rule_test!(test_scrub_span_pii_string_rules_password, "@password", "my_password_123", @r###" - { - "@password|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } - } - "###); // @password defaults to remove - attribute_rule_test!(test_scrub_span_pii_string_rules_password_remove, "@password:remove", "my_password_123", @r###" - { - "@password:remove|my_password_123": { - "type": "string", - "value": "" - }, - "_meta": { - "@password:remove|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:remove", - "x", - 0, - 0 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_password_replace, "@password:replace", "my_password_123", @r###" - { - "@password:replace|my_password_123": { - "type": "string", - "value": "[password]" - }, - "_meta": { - "@password:replace|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:replace", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - attribute_rule_test!(test_scrub_span_pii_string_rules_password_mask, "@password:mask", "my_password_123", @r###" - { - "@password:mask|my_password_123": { - "type": "string", - "value": "***************" - }, - "_meta": { - "@password:mask|my_password_123": { - "value": { - "": { - "rem": [ - [ - "@password:mask", - "m", - 0, - 15 - ] - ], - "len": 15 - } - } - } - } - } - "###); - - // Bearer token rules - attribute_rule_test!(test_scrub_span_pii_string_rules_bearer, "@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", @r###" - { - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } - } - "###); - - attribute_rule_test!( - test_scrub_span_pii_string_rules_bearer_replace, - "@bearer:replace", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - @r###" - { - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "Bearer [token]" - }, - "_meta": { - "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:replace", - "s", - 0, - 14 - ] - ], - "len": 43 - } - } - } - } - } - "###); - - attribute_rule_test!( - test_scrub_span_pii_string_rules_bearer_remove, - "@bearer:remove", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - @r###" - { - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "" - }, - "_meta": { - "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:remove", - "x", - 0, - 0 - ] - ], - "len": 43 - } - } - } - } - } - "###); - - attribute_rule_test!( - test_scrub_span_pii_string_rules_bearer_mask, - "@bearer:mask", - "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", - @r###" - { - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "type": "string", - "value": "*******************************************" - }, - "_meta": { - "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": { - "value": { - "": { - "rem": [ - [ - "@bearer:mask", - "m", - 0, - 43 - ] - ], - "len": 43 - } - } - } - } - } - "###); - #[test] fn test_scrub_span_implicit_attribute_value_does_not_match() { let json = r#" From 38977dd8162d07761797a533640685321392073d Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 14 Oct 2025 15:12:54 +0200 Subject: [PATCH 10/20] Move more tests --- relay-pii/src/eap.rs | 1114 +++++++++++++++- relay-server/src/processing/logs/process.rs | 1114 ---------------- relay-server/src/processing/spans/process.rs | 1251 +----------------- 3 files changed, 1159 insertions(+), 2320 deletions(-) diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index 393c0d34608..d8abe69a5b4 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -33,10 +33,1120 @@ pub fn scrub_eap_item( #[cfg(test)] mod tests { use relay_event_schema::processor::ValueType; - use relay_event_schema::protocol::{OurLog, SpanV2}; + use relay_event_schema::protocol::{Attributes, OurLog, SpanV2}; use relay_protocol::{Annotated, SerializableAnnotated}; - use crate::PiiConfig; + use crate::{DataScrubbingConfig, PiiConfig, scrub_eap_item}; + + #[test] + fn test_scrub_attributes_pii_default_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `user.name` is the only one that should be scrubbed by default rules. + let json = r#"{ + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "secret123" + }, + "secret": { + "type": "string", + "value": "topsecret" + }, + "api_key": { + "type": "string", + "value": "sk-1234567890abcdef" + }, + "auth_token": { + "type": "string", + "value": "bearer_token_123" + }, + "credit_card": { + "type": "string", + "value": "4571234567890111" + }, + "visa_card": { + "type": "string", + "value": "4571 2345 6789 0111" + }, + "bank_account": { + "type": "string", + "value": "DE89370400440532013000" + }, + "iban_code": { + "type": "string", + "value": "NO9386011117945" + }, + "authorization": { + "type": "string", + "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" + }, + "private_key": { + "type": "string", + "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" + }, + "database_url": { + "type": "string", + "value": "https://username:password@example.com/db" + }, + "file_path": { + "type": "string", + "value": "C:\\Users\\john\\Documents\\secret.txt" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "social_security": { + "type": "string", + "value": "078-05-1120" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "client_ip": { + "type": "string", + "value": "192.168.1.100" + }, + "ipv6_addr": { + "type": "string", + "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "very_sensitive_data": { + "type": "string", + "value": "very_sensitive_data" + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let data_scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + scrub_ip_addresses: true, + exclude_fields: vec!["public_data".to_owned()], + sensitive_fields: vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ], + ..Default::default() + }; + + let scrubbing_config = data_scrubbing_config.pii_config().unwrap(); + + scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "api_key": { + "type": "string", + "value": "[Filtered]" + }, + "auth_token": { + "type": "string", + "value": "[Filtered]" + }, + "authorization": { + "type": "string", + "value": "[Filtered]" + }, + "bank_account": { + "type": "string", + "value": "[Filtered]" + }, + "client_ip": { + "type": "string", + "value": "[ip]" + }, + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "database_url": { + "type": "string", + "value": "[Filtered]" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "file_path": { + "type": "string", + "value": "[Filtered]" + }, + "iban_code": { + "type": "string", + "value": "[Filtered]" + }, + "ipv6_addr": { + "type": "string", + "value": "[ip]" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "private_key": { + "type": "string", + "value": "[Filtered]" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "secret": { + "type": "string", + "value": "[Filtered]" + }, + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "social_security": { + "type": "string", + "value": "[Filtered]" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "very_sensitive_data": { + "type": "string", + "value": "[Filtered]" + }, + "visa_card": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "api_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "auth_token": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "authorization": { + "value": { + "": { + "rem": [ + [ + "@bearer:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + }, + "bank_account": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 22 + } + } + }, + "client_ip": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 13 + } + } + }, + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "database_url": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 40 + } + } + }, + "file_path": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 34 + } + } + }, + "iban_code": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + }, + "ipv6_addr": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 39 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "private_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 118 + } + } + }, + "secret": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "social_security": { + "value": { + "": { + "rem": [ + [ + "@usssn:filter", + "s", + 0, + 10 + ] + ], + "len": 11 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "very_sensitive_data": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "visa_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_pii_custom_object_rules() { + // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: + // * `user.name`: `true` + // * `sentry.release`: `false` + // * `url.path`: `maybe` + // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. + let json = r#" + { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "test_field_mac": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "test_field_imei": { + "type": "string", + "value": "356938035643809" + }, + "test_field_uuid": { + "type": "string", + "value": "123e4567-e89b-12d3-a456-426614174000" + }, + "test_field_regex_passes": { + "type": "string", + "value": "wxyz" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + } + }"#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: false, + scrub_ip_addresses: false, + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + let config = serde_json::from_value::(serde_json::json!( + { + "rules": { + "project:0": { + "type": "mac", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:1": { + "type": "imei", + "redaction": { + "method": "replace", + "text": "ITS_GONE" + } + }, + "project:2": { + "type": "uuid", + "redaction": { + "method": "replace", + "text": "BYE" + } + }, + "project:3": { + "type": "pattern", + "pattern": "[w-z]+", + "redaction": { + "method": "replace", + "text": "REGEXED" + } + }, + "project:4": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[USER NAME]" + } + }, + "project:5": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[RELEASE]" + } + }, + "project:6": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[URL PATH]" + } + }, + "project:7": { + "type": "anything", + "redaction": { + "method": "replace", + "text": "[DESCRIPTION]" + } + } + }, + "applications": { + "test_field_mac.value": [ + "project:0" + ], + "test_field_imei.value": [ + "project:1" + ], + "test_field_uuid.value": [ + "project:2" + ], + "test_field_regex_passes.value || test_field_regex_fails.value": [ + "project:3" + ], + "'user.name'.value": [ + "project:4" + ], + "'sentry.release'.value": [ + "project:5" + ], + "'url.path'.value": [ + "project:6" + ], + "'sentry.description'.value": [ + "project:7" + ] + } + } + )) + .unwrap(); + + scrub_eap_item( + ValueType::Span, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "password": { + "type": "string", + "value": "default_scrubbing_rules_are_off" + }, + "sentry.description": { + "type": "string", + "value": "[DESCRIPTION]" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "test_field_imei": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_mac": { + "type": "string", + "value": "ITS_GONE" + }, + "test_field_regex_fails": { + "type": "string", + "value": "abc" + }, + "test_field_regex_passes": { + "type": "string", + "value": "REGEXED" + }, + "test_field_uuid": { + "type": "string", + "value": "BYE" + }, + "url.path": { + "type": "string", + "value": "[URL PATH]" + }, + "user.name": { + "type": "string", + "value": "[USER NAME]" + }, + "_meta": { + "sentry.description": { + "value": { + "": { + "rem": [ + [ + "project:7", + "s", + 0, + 13 + ] + ], + "len": 9 + } + } + }, + "test_field_imei": { + "value": { + "": { + "rem": [ + [ + "project:1", + "s", + 0, + 8 + ] + ], + "len": 15 + } + } + }, + "test_field_mac": { + "value": { + "": { + "rem": [ + [ + "project:0", + "s", + 0, + 8 + ] + ], + "len": 17 + } + } + }, + "test_field_regex_passes": { + "value": { + "": { + "rem": [ + [ + "project:3", + "s", + 0, + 7 + ] + ], + "len": 4 + } + } + }, + "test_field_uuid": { + "value": { + "": { + "rem": [ + [ + "project:2", + "s", + 0, + 3 + ] + ], + "len": 36 + } + } + }, + "url.path": { + "value": { + "": { + "rem": [ + [ + "project:6", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "project:4", + "s", + 0, + 11 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_sensitive_fields() { + let json = r#" + { + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "should_be_removed" + }, + "another_sensitive": { + "type": "integer", + "value": 42 + }, + "my_value": { + "type": "string", + "value": "this_should_be_removed_as_sensitive" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: false, + scrub_ip_addresses: false, + sensitive_fields: vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "sensitive_custom".to_owned(), + "another_sensitive".to_owned(), + ], + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "another_sensitive": { + "type": "integer", + "value": null + }, + "my_value": { + "type": "string", + "value": "[Filtered]" + }, + "normal_field": { + "type": "string", + "value": "normal_data" + }, + "sensitive_custom": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "another_sensitive": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "x" + ] + ] + } + } + }, + "my_value": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 35 + } + } + }, + "sensitive_custom": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 17 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_safe_fields() { + let json = r#" + { + "password": { + "type": "string", + "value": "secret123" + }, + "credit_card": { + "type": "string", + "value": "4242424242424242" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + scrub_ip_addresses: false, + exclude_fields: vec!["secret".to_owned()], // Only 'secret' is safe + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "secret": { + "type": "string", + "value": "this_should_stay" + }, + "_meta": { + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + } + } + } + "###); + } + + #[test] + fn test_scrub_attributes_implicit_attribute_value() { + let json = r#" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + "#; + + let mut data = Annotated::::from_json(json).unwrap(); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match + } + })) + .unwrap(); + + let scrubbing_config = DataScrubbingConfig { + scrub_data: true, + scrub_defaults: true, + ..Default::default() + }; + + let scrubbing_config = scrubbing_config.pii_config().unwrap(); + + scrub_eap_item( + ValueType::OurLog, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc123" + } + } + "###); + + let config = serde_json::from_value::(serde_json::json!({ + "rules": { + "remove_abc123": { + "type": "pattern", + "pattern": "abc123", + "redaction": { + "method": "replace", + "text": "abc---" + } + }, + }, + "applications": { + "remove_this_string_abc123.value": ["remove_abc123"], + } + })) + .unwrap(); + + scrub_eap_item( + ValueType::OurLog, + &mut data, + Some(&config), + scrubbing_config.as_ref(), + ) + .unwrap(); + + insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" + { + "remove_this_string_abc123": { + "type": "string", + "value": "abc---" + }, + "_meta": { + "remove_this_string_abc123": { + "value": { + "": { + "rem": [ + [ + "remove_abc123", + "s", + 0, + 6 + ] + ], + "len": 6 + } + } + } + } + } + "###); + } macro_rules! attribute_rule_test { ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => { diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 32e6ebeaeb2..19a1bd09bd2 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -155,780 +155,6 @@ mod tests { } } - #[test] - fn test_scrub_log_pii_default_rules() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `user.name` is the only one that should be scrubbed by default rules. - let json = r#"{ - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log message with sensitive data: user@example.com and 4571234567890111", - "attributes": { - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "secret123" - }, - "secret": { - "type": "string", - "value": "topsecret" - }, - "api_key": { - "type": "string", - "value": "sk-1234567890abcdef" - }, - "auth_token": { - "type": "string", - "value": "bearer_token_123" - }, - "credit_card": { - "type": "string", - "value": "4571234567890111" - }, - "visa_card": { - "type": "string", - "value": "4571 2345 6789 0111" - }, - "bank_account": { - "type": "string", - "value": "DE89370400440532013000" - }, - "iban_code": { - "type": "string", - "value": "NO9386011117945" - }, - "authorization": { - "type": "string", - "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" - }, - "private_key": { - "type": "string", - "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" - }, - "database_url": { - "type": "string", - "value": "https://username:password@example.com/db" - }, - "file_path": { - "type": "string", - "value": "C:\\Users\\john\\Documents\\secret.txt" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "social_security": { - "type": "string", - "value": "078-05-1120" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "client_ip": { - "type": "string", - "value": "192.168.1.100" - }, - "ipv6_addr": { - "type": "string", - "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "very_sensitive_data": { - "type": "string", - "value": "very_sensitive_data" - } - } - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = true; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "very_sensitive_data".to_owned(), - ]; - scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; - - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "api_key": { - "type": "string", - "value": "[Filtered]" - }, - "auth_token": { - "type": "string", - "value": "[Filtered]" - }, - "authorization": { - "type": "string", - "value": "[Filtered]" - }, - "bank_account": { - "type": "string", - "value": "[Filtered]" - }, - "client_ip": { - "type": "string", - "value": "[ip]" - }, - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "database_url": { - "type": "string", - "value": "[Filtered]" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "file_path": { - "type": "string", - "value": "[Filtered]" - }, - "iban_code": { - "type": "string", - "value": "[Filtered]" - }, - "ipv6_addr": { - "type": "string", - "value": "[ip]" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "private_key": { - "type": "string", - "value": "[Filtered]" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "secret": { - "type": "string", - "value": "[Filtered]" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "social_security": { - "type": "string", - "value": "[Filtered]" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "[Filtered]" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "very_sensitive_data": { - "type": "string", - "value": "[Filtered]" - }, - "visa_card": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "api_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "auth_token": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "authorization": { - "value": { - "": { - "rem": [ - [ - "@bearer:filter", - "s", - 0, - 10 - ] - ], - "len": 43 - } - } - }, - "bank_account": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 22 - } - } - }, - "client_ip": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 13 - } - } - }, - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "database_url": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 40 - } - } - }, - "file_path": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 34 - } - } - }, - "iban_code": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - }, - "ipv6_addr": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 39 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "private_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 118 - } - } - }, - "secret": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "social_security": { - "value": { - "": { - "rem": [ - [ - "@usssn:filter", - "s", - 0, - 10 - ] - ], - "len": 11 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "very_sensitive_data": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "visa_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_pii_custom_object_rules() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log message with sensitive data: user@example.com and 4571234567890111", - "attributes": { - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "test_field_mac": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "test_field_imei": { - "type": "string", - "value": "356938035643809" - }, - "test_field_uuid": { - "type": "string", - "value": "123e4567-e89b-12d3-a456-426614174000" - }, - "test_field_regex_passes": { - "type": "string", - "value": "wxyz" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - } - } - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let config = serde_json::from_value::(serde_json::json!( - { - "rules": { - "project:0": { - "type": "mac", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:1": { - "type": "imei", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:2": { - "type": "uuid", - "redaction": { - "method": "replace", - "text": "BYE" - } - }, - "project:3": { - "type": "pattern", - "pattern": "[w-z]+", - "redaction": { - "method": "replace", - "text": "REGEXED" - } - }, - "project:4": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[USER NAME]" - } - }, - "project:5": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[RELEASE]" - } - }, - "project:6": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[URL PATH]" - } - } - }, - "applications": { - "test_field_mac.value": [ - "project:0" - ], - "test_field_imei.value": [ - "project:1" - ], - "test_field_uuid.value": [ - "project:2" - ], - "test_field_regex_passes.value || test_field_regex_fails.value": [ - "project:3" - ], - "'user.name'.value": [ - "project:4" - ], - "'sentry.release'.value": [ - "project:5" - ], - "'url.path'.value": [ - "project:6" - ] - } - } - )) - .unwrap(); - - let ctx = make_context(scrubbing_config, Some(config)); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "test_field_imei": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_mac": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - }, - "test_field_regex_passes": { - "type": "string", - "value": "REGEXED" - }, - "test_field_uuid": { - "type": "string", - "value": "BYE" - }, - "url.path": { - "type": "string", - "value": "[URL PATH]" - }, - "user.name": { - "type": "string", - "value": "[USER NAME]" - }, - "_meta": { - "test_field_imei": { - "value": { - "": { - "rem": [ - [ - "project:1", - "s", - 0, - 8 - ] - ], - "len": 15 - } - } - }, - "test_field_mac": { - "value": { - "": { - "rem": [ - [ - "project:0", - "s", - 0, - 8 - ] - ], - "len": 17 - } - } - }, - "test_field_regex_passes": { - "value": { - "": { - "rem": [ - [ - "project:3", - "s", - 0, - 7 - ] - ], - "len": 4 - } - } - }, - "test_field_uuid": { - "value": { - "": { - "rem": [ - [ - "project:2", - "s", - 0, - 3 - ] - ], - "len": 36 - } - } - }, - "url.path": { - "value": { - "": { - "rem": [ - [ - "project:6", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "project:4", - "s", - 0, - 11 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } - #[test] fn test_scrub_log_base_fields() { let json = r#" @@ -959,346 +185,6 @@ mod tests { insta::assert_json_snapshot!(SerializableAnnotated(&data)); } - #[test] - fn test_scrub_log_implicit_attribute_value_does_not_match() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$log.attributes.remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "###); - } - - #[test] - fn test_scrub_log_explicit_attribute_value_does_match() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$log.attributes.remove_this_string_abc123.value": ["remove_abc123"], - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc---" - } - }, - "_meta": { - "attributes": { - "remove_this_string_abc123": { - "value": { - "": { - "rem": [ - [ - "remove_abc123", - "s", - 0, - 6 - ] - ], - "len": 6 - } - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_sensitive_fields() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log", - "attributes": { - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "should_be_removed" - }, - "another_sensitive": { - "type": "integer", - "value": 42 - }, - "my_value": { - "type": "string", - "value": "this_should_be_removed_as_sensitive" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "sensitive_custom".to_owned(), - "another_sensitive".to_owned(), - ]; - - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "another_sensitive": { - "type": "integer", - "value": null - }, - "my_value": { - "type": "string", - "value": "[Filtered]" - }, - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "another_sensitive": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "x" - ] - ] - } - } - }, - "my_value": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 35 - } - } - }, - "sensitive_custom": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 17 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_log_safe_fields() { - let json = r#" - { - "timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "level": "info", - "body": "Test log with email@example.com", - "attributes": { - "password": { - "type": "string", - "value": "secret123" - }, - "credit_card": { - "type": "string", - "value": "4242424242424242" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.exclude_fields = vec!["secret".to_owned()]; // Only 'secret' is safe - let ctx = make_context(scrubbing_config, None); - scrub_log(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - }, - "_meta": { - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } - #[test] fn test_scrub_log_deep_wild_cards() { let json = r#" diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index ae5499cfea7..3d76cb05939 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -152,7 +152,7 @@ mod tests { } #[test] - fn test_scrub_span_pii_default_rules() { + fn test_scrub_span_pii_default_rules_links() { // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: // * `user.name`: `true` // * `sentry.release`: `false` @@ -164,112 +164,24 @@ mod tests { "trace_id": "5b8efff798038103d269b633813fc60c", "span_id": "eee19b7ec3c1b174", "name": "test", - "attributes": { - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "secret123" - }, - "secret": { - "type": "string", - "value": "topsecret" - }, - "api_key": { - "type": "string", - "value": "sk-1234567890abcdef" - }, - "auth_token": { - "type": "string", - "value": "bearer_token_123" - }, - "credit_card": { - "type": "string", - "value": "4571234567890111" - }, - "visa_card": { - "type": "string", - "value": "4571 2345 6789 0111" - }, - "bank_account": { - "type": "string", - "value": "DE89370400440532013000" - }, - "iban_code": { - "type": "string", - "value": "NO9386011117945" - }, - "authorization": { - "type": "string", - "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9" - }, - "private_key": { - "type": "string", - "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----" - }, - "database_url": { - "type": "string", - "value": "https://username:password@example.com/db" - }, - "file_path": { - "type": "string", - "value": "C:\\Users\\john\\Documents\\secret.txt" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "social_security": { - "type": "string", - "value": "078-05-1120" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "client_ip": { - "type": "string", - "value": "192.168.1.100" - }, - "ipv6_addr": { - "type": "string", - "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "very_sensitive_data": { - "type": "string", - "value": "very_sensitive_data" + "links": [{ + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + } } - } + }] }"#; let mut data = Annotated::::from_json(json).unwrap(); @@ -278,771 +190,42 @@ mod tests { scrubbing_config.scrub_data = true; scrubbing_config.scrub_defaults = true; scrubbing_config.scrub_ip_addresses = true; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "very_sensitive_data".to_owned(), - ]; - scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; - - let ctx = make_context(scrubbing_config, None); - scrub_span(&mut data, ctx).unwrap(); - - let span = data.value().unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&span.attributes), @r###" - { - "api_key": { - "type": "string", - "value": "[Filtered]" - }, - "auth_token": { - "type": "string", - "value": "[Filtered]" - }, - "authorization": { - "type": "string", - "value": "[Filtered]" - }, - "bank_account": { - "type": "string", - "value": "[Filtered]" - }, - "client_ip": { - "type": "string", - "value": "[ip]" - }, - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "database_url": { - "type": "string", - "value": "[Filtered]" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "file_path": { - "type": "string", - "value": "[Filtered]" - }, - "iban_code": { - "type": "string", - "value": "[Filtered]" - }, - "ipv6_addr": { - "type": "string", - "value": "[ip]" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "private_key": { - "type": "string", - "value": "[Filtered]" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "secret": { - "type": "string", - "value": "[Filtered]" - }, - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "social_security": { - "type": "string", - "value": "[Filtered]" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "[Filtered]" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "very_sensitive_data": { - "type": "string", - "value": "[Filtered]" - }, - "visa_card": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "api_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "auth_token": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "authorization": { - "value": { - "": { - "rem": [ - [ - "@bearer:filter", - "s", - 0, - 10 - ] - ], - "len": 43 - } - } - }, - "bank_account": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 22 - } - } - }, - "client_ip": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 13 - } - } - }, - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "database_url": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 40 - } - } - }, - "file_path": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 34 - } - } - }, - "iban_code": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - }, - "ipv6_addr": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 39 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "private_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 118 - } - } - }, - "secret": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "social_security": { - "value": { - "": { - "rem": [ - [ - "@usssn:filter", - "s", - 0, - 10 - ] - ], - "len": 11 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "very_sensitive_data": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "visa_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_span_pii_default_rules_links() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `user.name` is the only one that should be scrubbed by default rules. - let json = r#"{ - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "links": [{ - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "attributes": { - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - } - } - }] - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = true; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "very_sensitive_data".to_owned(), - ]; - scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; - - let ctx = make_context(scrubbing_config, None); - scrub_span(&mut data, ctx).unwrap(); - - let link = data.value().unwrap().links.value().unwrap()[0] - .value() - .unwrap(); - insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" - { - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "user.name": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_span_pii_custom_object_rules() { - // `user.name`, `sentry.release`, and `url.path` are marked as follows in `sentry-conventions`: - // * `user.name`: `true` - // * `sentry.release`: `false` - // * `url.path`: `maybe` - // Therefore, `sentry.release` is the only one that should not be scrubbed by custom rules. - let json = r#" - { - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": { - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "test_field_mac": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "test_field_imei": { - "type": "string", - "value": "356938035643809" - }, - "test_field_uuid": { - "type": "string", - "value": "123e4567-e89b-12d3-a456-426614174000" - }, - "test_field_regex_passes": { - "type": "string", - "value": "wxyz" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - } - } - }"#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - - let config = serde_json::from_value::(serde_json::json!( - { - "rules": { - "project:0": { - "type": "mac", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:1": { - "type": "imei", - "redaction": { - "method": "replace", - "text": "ITS_GONE" - } - }, - "project:2": { - "type": "uuid", - "redaction": { - "method": "replace", - "text": "BYE" - } - }, - "project:3": { - "type": "pattern", - "pattern": "[w-z]+", - "redaction": { - "method": "replace", - "text": "REGEXED" - } - }, - "project:4": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[USER NAME]" - } - }, - "project:5": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[RELEASE]" - } - }, - "project:6": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[URL PATH]" - } - }, - "project:7": { - "type": "anything", - "redaction": { - "method": "replace", - "text": "[DESCRIPTION]" - } - } - }, - "applications": { - "test_field_mac.value": [ - "project:0" - ], - "test_field_imei.value": [ - "project:1" - ], - "test_field_uuid.value": [ - "project:2" - ], - "test_field_regex_passes.value || test_field_regex_fails.value": [ - "project:3" - ], - "'user.name'.value": [ - "project:4" - ], - "'sentry.release'.value": [ - "project:5" - ], - "'url.path'.value": [ - "project:6" - ], - "'sentry.description'.value": [ - "project:7" - ] - } - } - )) - .unwrap(); - - let ctx = make_context(scrubbing_config, Some(config)); - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "password": { - "type": "string", - "value": "default_scrubbing_rules_are_off" - }, - "sentry.description": { - "type": "string", - "value": "[DESCRIPTION]" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "test_field_imei": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_mac": { - "type": "string", - "value": "ITS_GONE" - }, - "test_field_regex_fails": { - "type": "string", - "value": "abc" - }, - "test_field_regex_passes": { - "type": "string", - "value": "REGEXED" - }, - "test_field_uuid": { - "type": "string", - "value": "BYE" - }, - "url.path": { - "type": "string", - "value": "[URL PATH]" - }, - "user.name": { - "type": "string", - "value": "[USER NAME]" - }, - "_meta": { - "sentry.description": { - "value": { - "": { - "rem": [ - [ - "project:7", - "s", - 0, - 13 - ] - ], - "len": 9 - } - } - }, - "test_field_imei": { - "value": { - "": { - "rem": [ - [ - "project:1", - "s", - 0, - 8 - ] - ], - "len": 15 - } - } - }, - "test_field_mac": { - "value": { - "": { - "rem": [ - [ - "project:0", - "s", - 0, - 8 - ] - ], - "len": 17 - } - } - }, - "test_field_regex_passes": { - "value": { - "": { - "rem": [ - [ - "project:3", - "s", - 0, - 7 - ] - ], - "len": 4 - } - } - }, - "test_field_uuid": { - "value": { - "": { - "rem": [ - [ - "project:2", - "s", - 0, - 3 - ] - ], - "len": 36 - } - } - }, - "url.path": { - "value": { - "": { - "rem": [ - [ - "project:6", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, + scrubbing_config.sensitive_fields = vec![ + "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. + "very_sensitive_data".to_owned(), + ]; + scrubbing_config.exclude_fields = vec!["public_data".to_owned()]; + + let ctx = make_context(scrubbing_config, None); + scrub_span(&mut data, ctx).unwrap(); + + let link = data.value().unwrap().links.value().unwrap()[0] + .value() + .unwrap(); + insta::assert_json_snapshot!(SerializableAnnotated(&link.attributes), @r###" + { + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { "user.name": { "value": { "": { "rem": [ [ - "project:4", + "@password:filter", "s", 0, - 11 + 10 ] ], "len": 9 @@ -1370,344 +553,4 @@ mod tests { } "###); } - - #[test] - fn test_scrub_span_implicit_attribute_value_does_not_match() { - let json = r#" - { - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$span.attributes.remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "###); - } - - #[test] - fn test_scrub_span_explicit_attribute_value_does_match() { - let json = r#" - { - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc123" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let config = serde_json::from_value::(serde_json::json!({ - "rules": { - "remove_abc123": { - "type": "pattern", - "pattern": "abc123", - "redaction": { - "method": "replace", - "text": "abc---" - } - }, - }, - "applications": { - "$span.attributes.remove_this_string_abc123.value": ["remove_abc123"], - } - })) - .unwrap(); - - let mut scrubbing_config = relay_pii::DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - - let ctx = make_context(scrubbing_config, Some(config)); - - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "attributes": { - "remove_this_string_abc123": { - "type": "string", - "value": "abc---" - } - }, - "_meta": { - "attributes": { - "remove_this_string_abc123": { - "value": { - "": { - "rem": [ - [ - "remove_abc123", - "s", - 0, - 6 - ] - ], - "len": 6 - } - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_span_sensitive_fields() { - let json = r#" - { - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": { - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "should_be_removed" - }, - "another_sensitive": { - "type": "integer", - "value": 42 - }, - "my_value": { - "type": "string", - "value": "this_should_be_removed_as_sensitive" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = false; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.sensitive_fields = vec![ - "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed. - "sensitive_custom".to_owned(), - "another_sensitive".to_owned(), - ]; - - let ctx = make_context(scrubbing_config, None); - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "another_sensitive": { - "type": "integer", - "value": null - }, - "my_value": { - "type": "string", - "value": "[Filtered]" - }, - "normal_field": { - "type": "string", - "value": "normal_data" - }, - "sensitive_custom": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "another_sensitive": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "x" - ] - ] - } - } - }, - "my_value": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 35 - } - } - }, - "sensitive_custom": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 17 - } - } - } - } - } - "###); - } - - #[test] - fn test_scrub_span_safe_fields() { - let json = r#" - { - "start_timestamp": 1544719859.0, - "end_timestamp": 1544719860.0, - "trace_id": "5b8efff798038103d269b633813fc60c", - "span_id": "eee19b7ec3c1b174", - "name": "test", - "attributes": { - "password": { - "type": "string", - "value": "secret123" - }, - "credit_card": { - "type": "string", - "value": "4242424242424242" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - } - } - } - "#; - - let mut data = Annotated::::from_json(json).unwrap(); - - let mut scrubbing_config = DataScrubbingConfig::default(); - scrubbing_config.scrub_data = true; - scrubbing_config.scrub_defaults = true; - scrubbing_config.scrub_ip_addresses = false; - scrubbing_config.exclude_fields = vec!["secret".to_owned()]; // Only 'secret' is safe - let ctx = make_context(scrubbing_config, None); - scrub_span(&mut data, ctx).unwrap(); - - insta::assert_json_snapshot!(SerializableAnnotated(&data.value().unwrap().attributes), @r###" - { - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "secret": { - "type": "string", - "value": "this_should_stay" - }, - "_meta": { - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - } - } - } - "###); - } } From e218783d2e5c928b94dfb56777efebb6eee31a56 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 14 Oct 2025 15:58:47 +0200 Subject: [PATCH 11/20] Add docstring --- relay-pii/src/eap.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index d8abe69a5b4..9f906fec509 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -5,6 +5,7 @@ use relay_protocol::Annotated; use crate::{AttributeMode, PiiConfig, PiiProcessor}; +/// Scrubs an EAP item (such as an `OurLog` or `SpanV2`) with the given PII configs. pub fn scrub_eap_item( value_type: ValueType, item: &mut Annotated, From f48edc168118d6797eb1960c57ea1d4ec79baddf Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 09:51:11 +0200 Subject: [PATCH 12/20] Rename scrub_eap_item to scrub --- relay-pii/src/eap.rs | 24 ++++++++++++-------- relay-pii/src/lib.rs | 3 +-- relay-server/src/processing/logs/process.rs | 2 +- relay-server/src/processing/spans/process.rs | 2 +- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index 9f906fec509..02a7a893933 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -1,3 +1,7 @@ +//! Provides the `scrub` function for scrubbing EAP items. +//! +//! This module also contains tests for scrubbing attributes. + use relay_event_schema::processor::{ ProcessValue, ProcessingResult, ProcessingState, ValueType, process_value, }; @@ -6,7 +10,7 @@ use relay_protocol::Annotated; use crate::{AttributeMode, PiiConfig, PiiProcessor}; /// Scrubs an EAP item (such as an `OurLog` or `SpanV2`) with the given PII configs. -pub fn scrub_eap_item( +pub fn scrub( value_type: ValueType, item: &mut Annotated, pii_config: Option<&PiiConfig>, @@ -37,7 +41,7 @@ mod tests { use relay_event_schema::protocol::{Attributes, OurLog, SpanV2}; use relay_protocol::{Annotated, SerializableAnnotated}; - use crate::{DataScrubbingConfig, PiiConfig, scrub_eap_item}; + use crate::{DataScrubbingConfig, PiiConfig, eap}; #[test] fn test_scrub_attributes_pii_default_rules() { @@ -169,7 +173,7 @@ mod tests { let scrubbing_config = data_scrubbing_config.pii_config().unwrap(); - scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" { @@ -691,7 +695,7 @@ mod tests { )) .unwrap(); - scrub_eap_item( + eap::scrub( ValueType::Span, &mut data, Some(&config), @@ -891,7 +895,7 @@ mod tests { let scrubbing_config = scrubbing_config.pii_config().unwrap(); - scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" { @@ -990,7 +994,7 @@ mod tests { let scrubbing_config = scrubbing_config.pii_config().unwrap(); - scrub_eap_item(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); + eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" { @@ -1080,7 +1084,7 @@ mod tests { let scrubbing_config = scrubbing_config.pii_config().unwrap(); - scrub_eap_item( + eap::scrub( ValueType::OurLog, &mut data, Some(&config), @@ -1114,7 +1118,7 @@ mod tests { })) .unwrap(); - scrub_eap_item( + eap::scrub( ValueType::OurLog, &mut data, Some(&config), @@ -1189,7 +1193,7 @@ mod tests { let mut span = Annotated::::from_json(&span_json).unwrap(); - crate::scrub_eap_item(ValueType::Span, &mut span, Some(&config), scrubbing_config.as_ref()).unwrap(); + eap::scrub(ValueType::Span, &mut span, Some(&config), scrubbing_config.as_ref()).unwrap(); insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&span.value().unwrap().attributes), @$snapshot)); @@ -1215,7 +1219,7 @@ mod tests { let mut log = Annotated::::from_json(&log_json).unwrap(); - crate::scrub_eap_item(ValueType::OurLog, &mut log, Some(&config), scrubbing_config.as_ref()).unwrap(); + eap::scrub(ValueType::OurLog, &mut log, Some(&config), scrubbing_config.as_ref()).unwrap(); insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&log.value().unwrap().attributes), @$snapshot)); } diff --git a/relay-pii/src/lib.rs b/relay-pii/src/lib.rs index d4d5e52c9c5..efc02d70e17 100644 --- a/relay-pii/src/lib.rs +++ b/relay-pii/src/lib.rs @@ -11,7 +11,6 @@ mod builtin; mod compiledconfig; mod config; mod convert; -mod eap; mod generate_selectors; mod json; mod legacy; @@ -22,12 +21,12 @@ mod regexes; mod selector; mod utils; +pub mod eap; pub mod transform; pub use self::attachments::*; pub use self::compiledconfig::*; pub use self::config::*; -pub use self::eap::*; pub use self::generate_selectors::selector_suggestions_from_value; pub use self::json::*; pub use self::legacy::*; diff --git a/relay-server/src/processing/logs/process.rs b/relay-server/src/processing/logs/process.rs index 19a1bd09bd2..22131e93809 100644 --- a/relay-server/src/processing/logs/process.rs +++ b/relay-server/src/processing/logs/process.rs @@ -97,7 +97,7 @@ fn scrub_log(log: &mut Annotated, ctx: Context<'_>) -> Result<()> { .pii_config() .map_err(|e| Error::PiiConfig(e.clone()))?; - relay_pii::scrub_eap_item( + relay_pii::eap::scrub( ValueType::OurLog, log, ctx.project_info.config.pii_config.as_ref(), diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index 3d76cb05939..c6e68fc69aa 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -107,7 +107,7 @@ fn scrub_span(span: &mut Annotated, ctx: Context<'_>) -> Result<()> { .pii_config() .map_err(|e| Error::PiiConfig(e.clone()))?; - relay_pii::scrub_eap_item( + relay_pii::eap::scrub( ValueType::Span, span, ctx.project_info.config.pii_config.as_ref(), From 5899505be619e687d85e1708a2d592dbea34279b Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 10:51:15 +0200 Subject: [PATCH 13/20] Extract pytest fixtures --- tests/integration/conftest.py | 4 ++ tests/integration/fixtures/pii.py | 61 +++++++++++++++++++++++++++++++ tests/integration/test_ourlogs.py | 49 ++----------------------- tests/integration/test_spansv2.py | 49 ++----------------------- 4 files changed, 73 insertions(+), 90 deletions(-) create mode 100644 tests/integration/fixtures/pii.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 283a7ad9061..473b1969a82 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -18,6 +18,10 @@ get_relay_binary, latest_relay_version, ) +from .fixtures.pii import ( # noqa + scrubbing_rule, + secret_attribute, +) from .fixtures.processing import ( # noqa kafka_consumer, get_topic_name, diff --git a/tests/integration/fixtures/pii.py b/tests/integration/fixtures/pii.py new file mode 100644 index 00000000000..7927e65d711 --- /dev/null +++ b/tests/integration/fixtures/pii.py @@ -0,0 +1,61 @@ +import pytest + + +@pytest.fixture( + params=[ + ("@ip", "127.0.0.1", "[ip]"), + ("@email", "test@example.com", "[email]"), + ("@creditcard", "4242424242424242", "[creditcard]"), + ("@iban", "DE89370400440532013000", "[iban]"), + ("@mac", "4a:00:04:10:9b:50", "*****************"), + ( + "@uuid", + "ceee0822-ed8f-4622-b2a3-789e73e75cd1", + "************************************", + ), + ("@imei", "356938035643809", "[imei]"), + ( + "@pemkey", + "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", + "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", + ), + ( + "@urlauth", + "https://username:password@example.com/", + "https://[auth]@example.com/", + ), + ("@usssn", "078-05-1120", "***********"), + ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), + ("@password", "my_password_123", ""), + ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), + ], + ids=[ + "ip", + "email", + "creditard", + "iban", + "mac", + "uuid", + "imei", + "pemkey", + "urlauth", + "usssn", + "userpath", + "password", + "bearer", + ], +) +def scrubbing_rule(request): + return request.param + + +@pytest.fixture( + params=[ + ("password", "my_password_123", "[Filtered]", "@password:filter"), + ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), + ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), + ], + ids=["password", "secret_key", "api_key"], +) +def secret_attribute(request): + return request.param diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index b18a315eec4..1e1408d261c 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -305,43 +305,12 @@ def test_ourlog_extraction_with_sentry_logs( ] -@pytest.mark.parametrize( - "rule_type,test_value,expected_scrubbed", - [ - ("@ip", "127.0.0.1", "[ip]"), - ("@email", "test@example.com", "[email]"), - ("@creditcard", "4242424242424242", "[creditcard]"), - ("@iban", "DE89370400440532013000", "[iban]"), - ("@mac", "4a:00:04:10:9b:50", "*****************"), - ( - "@uuid", - "ceee0822-ed8f-4622-b2a3-789e73e75cd1", - "************************************", - ), - ("@imei", "356938035643809", "[imei]"), - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", - ), - ( - "@urlauth", - "https://username:password@example.com/", - "https://[auth]@example.com/", - ), - ("@usssn", "078-05-1120", "***********"), - ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), - ("@password", "my_password_123", ""), - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), - ], -) def test_ourlog_extraction_with_string_pii_scrubbing( mini_sentry, relay, - rule_type, - test_value, - expected_scrubbed, + scrubbing_rule, ): + rule_type, test_value, expected_scrubbed = scrubbing_rule project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["retentions"] = { @@ -387,23 +356,13 @@ def test_ourlog_extraction_with_string_pii_scrubbing( assert rule_type in rem_info[0] -@pytest.mark.parametrize( - "attribute_key,attribute_value,expected_value,rule_type", - [ - ("password", "my_password_123", "[Filtered]", "@password:filter"), - ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), - ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), - ], -) def test_ourlog_extraction_default_pii_scrubbing_attributes( mini_sentry, relay, items_consumer, - attribute_key, - attribute_value, - expected_value, - rule_type, + secret_attribute, ): + attribute_key, attribute_value, expected_value, rule_type = secret_attribute project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["features"] = [ diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index 26b77096a4e..0dc9ecb4195 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -540,43 +540,12 @@ def test_spanv2_inbound_filters( assert mini_sentry.captured_events.empty() -@pytest.mark.parametrize( - "rule_type,test_value,expected_scrubbed", - [ - ("@ip", "127.0.0.1", "[ip]"), - ("@email", "test@example.com", "[email]"), - ("@creditcard", "4242424242424242", "[creditcard]"), - ("@iban", "DE89370400440532013000", "[iban]"), - ("@mac", "4a:00:04:10:9b:50", "*****************"), - ( - "@uuid", - "ceee0822-ed8f-4622-b2a3-789e73e75cd1", - "************************************", - ), - ("@imei", "356938035643809", "[imei]"), - ( - "@pemkey", - "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----", - "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----", - ), - ( - "@urlauth", - "https://username:password@example.com/", - "https://[auth]@example.com/", - ), - ("@usssn", "078-05-1120", "***********"), - ("@userpath", "/Users/john/Documents", "/Users/[user]/Documents"), - ("@password", "my_password_123", ""), - ("@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", "Bearer [token]"), - ], -) def test_spanv2_with_string_pii_scrubbing( mini_sentry, relay, - rule_type, - test_value, - expected_scrubbed, + scrubbing_rule, ): + rule_type, test_value, expected_scrubbed = scrubbing_rule project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["features"] = [ @@ -621,22 +590,12 @@ def test_spanv2_with_string_pii_scrubbing( assert rule_type in rem_info[0] -@pytest.mark.parametrize( - "attribute_key,attribute_value,expected_value,rule_type", - [ - ("password", "my_password_123", "[Filtered]", "@password:filter"), - ("secret_key", "my_secret_key_123", "[Filtered]", "@password:filter"), - ("api_key", "my_api_key_123", "[Filtered]", "@password:filter"), - ], -) def test_spanv2_default_pii_scrubbing_attributes( mini_sentry, relay, - attribute_key, - attribute_value, - expected_value, - rule_type, + secret_attribute, ): + attribute_key, attribute_value, expected_value, rule_type = secret_attribute project_id = 42 project_config = mini_sentry.add_full_project_config(project_id) project_config["config"]["features"] = [ From f63156774dbb91d2108558a367ab261954914b50 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 10:52:53 +0200 Subject: [PATCH 14/20] Extract a snapshot --- relay-pii/src/eap.rs | 366 +---------------- ...s__scrub_attributes_pii_default_rules.snap | 367 ++++++++++++++++++ 2 files changed, 368 insertions(+), 365 deletions(-) create mode 100644 relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index 02a7a893933..a531a224588 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -175,371 +175,7 @@ mod tests { eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap(); - insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###" - { - "api_key": { - "type": "string", - "value": "[Filtered]" - }, - "auth_token": { - "type": "string", - "value": "[Filtered]" - }, - "authorization": { - "type": "string", - "value": "[Filtered]" - }, - "bank_account": { - "type": "string", - "value": "[Filtered]" - }, - "client_ip": { - "type": "string", - "value": "[ip]" - }, - "credit_card": { - "type": "string", - "value": "[Filtered]" - }, - "database_url": { - "type": "string", - "value": "[Filtered]" - }, - "device_imei": { - "type": "string", - "value": "356938035643809" - }, - "file_path": { - "type": "string", - "value": "[Filtered]" - }, - "iban_code": { - "type": "string", - "value": "[Filtered]" - }, - "ipv6_addr": { - "type": "string", - "value": "[ip]" - }, - "mac_address": { - "type": "string", - "value": "4a:00:04:10:9b:50" - }, - "password": { - "type": "string", - "value": "[Filtered]" - }, - "private_key": { - "type": "string", - "value": "[Filtered]" - }, - "public_data": { - "type": "string", - "value": "public_data" - }, - "secret": { - "type": "string", - "value": "[Filtered]" - }, - "sentry.description": { - "type": "string", - "value": "secret123" - }, - "sentry.release": { - "type": "string", - "value": "secret123" - }, - "session_id": { - "type": "string", - "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" - }, - "social_security": { - "type": "string", - "value": "[Filtered]" - }, - "unix_path": { - "type": "string", - "value": "/home/alice/private/data.json" - }, - "url.path": { - "type": "string", - "value": "secret123" - }, - "user.name": { - "type": "string", - "value": "[Filtered]" - }, - "user_email": { - "type": "string", - "value": "john.doe@example.com" - }, - "very_sensitive_data": { - "type": "string", - "value": "[Filtered]" - }, - "visa_card": { - "type": "string", - "value": "[Filtered]" - }, - "_meta": { - "api_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "auth_token": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "authorization": { - "value": { - "": { - "rem": [ - [ - "@bearer:filter", - "s", - 0, - 10 - ] - ], - "len": 43 - } - } - }, - "bank_account": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 22 - } - } - }, - "client_ip": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 13 - } - } - }, - "credit_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 16 - } - } - }, - "database_url": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 40 - } - } - }, - "file_path": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 34 - } - } - }, - "iban_code": { - "value": { - "": { - "rem": [ - [ - "@iban:filter", - "s", - 0, - 10 - ] - ], - "len": 15 - } - } - }, - "ipv6_addr": { - "value": { - "": { - "rem": [ - [ - "@ip:replace", - "s", - 0, - 4 - ] - ], - "len": 39 - } - } - }, - "password": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "private_key": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 118 - } - } - }, - "secret": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "social_security": { - "value": { - "": { - "rem": [ - [ - "@usssn:filter", - "s", - 0, - 10 - ] - ], - "len": 11 - } - } - }, - "user.name": { - "value": { - "": { - "rem": [ - [ - "@password:filter", - "s", - 0, - 10 - ] - ], - "len": 9 - } - } - }, - "very_sensitive_data": { - "value": { - "": { - "rem": [ - [ - "strip-fields", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - }, - "visa_card": { - "value": { - "": { - "rem": [ - [ - "@creditcard:filter", - "s", - 0, - 10 - ] - ], - "len": 19 - } - } - } - } - } - "###); + insta::assert_json_snapshot!(SerializableAnnotated(&data)); } #[test] diff --git a/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap b/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap new file mode 100644 index 00000000000..c959525916f --- /dev/null +++ b/relay-pii/src/snapshots/relay_pii__eap__tests__scrub_attributes_pii_default_rules.snap @@ -0,0 +1,367 @@ +--- +source: relay-pii/src/eap.rs +expression: SerializableAnnotated(&data) +--- +{ + "api_key": { + "type": "string", + "value": "[Filtered]" + }, + "auth_token": { + "type": "string", + "value": "[Filtered]" + }, + "authorization": { + "type": "string", + "value": "[Filtered]" + }, + "bank_account": { + "type": "string", + "value": "[Filtered]" + }, + "client_ip": { + "type": "string", + "value": "[ip]" + }, + "credit_card": { + "type": "string", + "value": "[Filtered]" + }, + "database_url": { + "type": "string", + "value": "[Filtered]" + }, + "device_imei": { + "type": "string", + "value": "356938035643809" + }, + "file_path": { + "type": "string", + "value": "[Filtered]" + }, + "iban_code": { + "type": "string", + "value": "[Filtered]" + }, + "ipv6_addr": { + "type": "string", + "value": "[ip]" + }, + "mac_address": { + "type": "string", + "value": "4a:00:04:10:9b:50" + }, + "password": { + "type": "string", + "value": "[Filtered]" + }, + "private_key": { + "type": "string", + "value": "[Filtered]" + }, + "public_data": { + "type": "string", + "value": "public_data" + }, + "secret": { + "type": "string", + "value": "[Filtered]" + }, + "sentry.description": { + "type": "string", + "value": "secret123" + }, + "sentry.release": { + "type": "string", + "value": "secret123" + }, + "session_id": { + "type": "string", + "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1" + }, + "social_security": { + "type": "string", + "value": "[Filtered]" + }, + "unix_path": { + "type": "string", + "value": "/home/alice/private/data.json" + }, + "url.path": { + "type": "string", + "value": "secret123" + }, + "user.name": { + "type": "string", + "value": "[Filtered]" + }, + "user_email": { + "type": "string", + "value": "john.doe@example.com" + }, + "very_sensitive_data": { + "type": "string", + "value": "[Filtered]" + }, + "visa_card": { + "type": "string", + "value": "[Filtered]" + }, + "_meta": { + "api_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "auth_token": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "authorization": { + "value": { + "": { + "rem": [ + [ + "@bearer:filter", + "s", + 0, + 10 + ] + ], + "len": 43 + } + } + }, + "bank_account": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 22 + } + } + }, + "client_ip": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 13 + } + } + }, + "credit_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 16 + } + } + }, + "database_url": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 40 + } + } + }, + "file_path": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 34 + } + } + }, + "iban_code": { + "value": { + "": { + "rem": [ + [ + "@iban:filter", + "s", + 0, + 10 + ] + ], + "len": 15 + } + } + }, + "ipv6_addr": { + "value": { + "": { + "rem": [ + [ + "@ip:replace", + "s", + 0, + 4 + ] + ], + "len": 39 + } + } + }, + "password": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "private_key": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 118 + } + } + }, + "secret": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "social_security": { + "value": { + "": { + "rem": [ + [ + "@usssn:filter", + "s", + 0, + 10 + ] + ], + "len": 11 + } + } + }, + "user.name": { + "value": { + "": { + "rem": [ + [ + "@password:filter", + "s", + 0, + 10 + ] + ], + "len": 9 + } + } + }, + "very_sensitive_data": { + "value": { + "": { + "rem": [ + [ + "strip-fields", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + }, + "visa_card": { + "value": { + "": { + "rem": [ + [ + "@creditcard:filter", + "s", + 0, + 10 + ] + ], + "len": 19 + } + } + } + } +} From 962694b87d8f523799d7c270c89296057b652e1b Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 10:55:14 +0200 Subject: [PATCH 15/20] Rename parameters --- relay-pii/src/eap.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/relay-pii/src/eap.rs b/relay-pii/src/eap.rs index a531a224588..e0daf48354c 100644 --- a/relay-pii/src/eap.rs +++ b/relay-pii/src/eap.rs @@ -13,19 +13,19 @@ use crate::{AttributeMode, PiiConfig, PiiProcessor}; pub fn scrub( value_type: ValueType, item: &mut Annotated, - pii_config: Option<&PiiConfig>, - pii_config_from_scrubbing: Option<&PiiConfig>, + advanced_rules: Option<&PiiConfig>, + legacy_rule: Option<&PiiConfig>, ) -> ProcessingResult { let state = ProcessingState::root().enter_borrowed("", None, [value_type]); - if let Some(config) = pii_config { + if let Some(config) = advanced_rules { let mut processor = PiiProcessor::new(config.compiled()) // For advanced rules we want to treat attributes as objects. .attribute_mode(AttributeMode::Object); process_value(item, &mut processor, &state)?; } - if let Some(config) = pii_config_from_scrubbing { + if let Some(config) = legacy_rule { let mut processor = PiiProcessor::new(config.compiled()) // For "legacy" rules we want to identify attributes with their values. .attribute_mode(AttributeMode::ValueOnly); From a3dad4ac2ef297087eccf33f5945eeb577f7288c Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 10:57:38 +0200 Subject: [PATCH 16/20] Move scrubbing after quota enforcement --- relay-server/src/processing/logs/mod.rs | 3 ++- relay-server/src/processing/spans/mod.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/relay-server/src/processing/logs/mod.rs b/relay-server/src/processing/logs/mod.rs index 74e2bba362d..e59b09e6458 100644 --- a/relay-server/src/processing/logs/mod.rs +++ b/relay-server/src/processing/logs/mod.rs @@ -154,10 +154,11 @@ impl processing::Processor for LogsProcessor { let mut logs = process::expand(logs); process::normalize(&mut logs); filter::filter(&mut logs, ctx); - process::scrub(&mut logs, ctx); self.limiter.enforce_quotas(&mut logs, ctx).await?; + process::scrub(&mut logs, ctx); + Ok(Output::just(LogOutput::Processed(logs))) } } diff --git a/relay-server/src/processing/spans/mod.rs b/relay-server/src/processing/spans/mod.rs index 4e627b90540..6f422072864 100644 --- a/relay-server/src/processing/spans/mod.rs +++ b/relay-server/src/processing/spans/mod.rs @@ -144,10 +144,11 @@ impl processing::Processor for SpansProcessor { process::normalize(&mut spans, &self.geo_lookup); filter::filter(&mut spans, ctx); - process::scrub(&mut spans, ctx); self.limiter.enforce_quotas(&mut spans, ctx).await?; + process::scrub(&mut spans, ctx); + let metrics = dynamic_sampling::create_indexed_metrics(&spans, ctx); Ok(Output { From 525ee9fd5fa5211c8f297b3bb9f70d709097c560 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 11:22:54 +0200 Subject: [PATCH 17/20] Refactor some more tests --- tests/integration/test_ourlogs.py | 39 +++++++++++++++++++++-------- tests/integration/test_spansv2.py | 41 +++++++++++++++++++++++-------- 2 files changed, 60 insertions(+), 20 deletions(-) diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 1e1408d261c..d1515fbcdff 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -343,17 +343,36 @@ def test_ourlog_extraction_with_string_pii_scrubbing( envelope = mini_sentry.captured_events.get() item_payload = json.loads(envelope.items[0].payload.bytes.decode()) item = item_payload["items"][0] - attributes = item["attributes"] - - assert "test_pii" in attributes - assert attributes["test_pii"]["value"] == expected_scrubbed - assert "_meta" in item - meta = item["_meta"]["attributes"]["test_pii"]["value"][""] - assert "rem" in meta - # Check that the rule type is mentioned in the metadata - rem_info = meta["rem"][0] - assert rule_type in rem_info[0] + assert item == { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "test_pii": { "type": "string", "value": expected_scrubbed}, + "sentry.browser.name": {"type": "string", "value": "Python Requests"}, + "sentry.browser.version": {"type": "string", "value": "2.32"}, + "sentry.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, + }, + "__header": {"byte_size": mock.ANY}, + "_meta": { + "attributes": { + "test_pii": { + "value": { + "": { + "len": mock.ANY, + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]] + } + } + } + }, + }, + "body": "Test log", + "level": "info", + "timestamp": time_within(ts), + } def test_ourlog_extraction_default_pii_scrubbing_attributes( diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index 0dc9ecb4195..e6b8c1e1778 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -577,17 +577,38 @@ def test_spanv2_with_string_pii_scrubbing( envelope = mini_sentry.captured_events.get() item_payload = json.loads(envelope.items[0].payload.bytes.decode()) item = item_payload["items"][0] - attributes = item["attributes"] - - assert "test_pii" in attributes - assert attributes["test_pii"]["value"] == expected_scrubbed - assert "_meta" in item - meta = item["_meta"]["attributes"]["test_pii"]["value"][""] - assert "rem" in meta - # Check that the rule type is mentioned in the metadata - rem_info = meta["rem"][0] - assert rule_type in rem_info[0] + assert item == { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "attributes": { + "test_pii": { "type": "string", "value": expected_scrubbed}, + "sentry.browser.name": {"type": "string", "value": "Python Requests"}, + "sentry.browser.version": {"type": "string", "value": "2.32"}, + "sentry.observed_timestamp_nanos": { + "type": "string", + "value": time_within(ts, expect_resolution="ns"), + }, + }, + "_meta": { + "attributes": { + "test_pii": { + "value": { + "": { + "len": mock.ANY, + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]] + } + } + } + }, + "status": {"": {"err": ["missing_attribute"]}}, + }, + "name": "Test span", + "start_timestamp": time_within(ts), + "end_timestamp": time_within(ts.timestamp() + 0.5), + "is_remote": False, + "status": None, + } def test_spanv2_default_pii_scrubbing_attributes( From 10be2634af01b99c5e65e24528f9436b86a5f057 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 13:58:06 +0200 Subject: [PATCH 18/20] Align spans::scrub with logs::scrub --- relay-server/src/processing/spans/process.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/relay-server/src/processing/spans/process.rs b/relay-server/src/processing/spans/process.rs index c6e68fc69aa..3a304de0f72 100644 --- a/relay-server/src/processing/spans/process.rs +++ b/relay-server/src/processing/spans/process.rs @@ -88,15 +88,13 @@ fn normalize_span( /// Applies PII scrubbing to individual spans. pub fn scrub(spans: &mut Managed, ctx: Context<'_>) { - spans.modify(|spans, records| { - spans.spans.retain_mut(|span| { - let r = scrub_span(span, ctx).inspect_err(|err| { - relay_log::debug!("failed to scrub pii from span: {err}"); - }); - - records.or_default(r.map(|_| true), &*span) - }) - }); + spans.retain( + |spans| &mut spans.spans, + |span, _| { + scrub_span(span, ctx) + .inspect_err(|err| relay_log::debug!("failed to scrub pii from span: {err}")) + }, + ); } fn scrub_span(span: &mut Annotated, ctx: Context<'_>) -> Result<()> { From b171c5e74d2960e7db0d553d43103cd10f3ed4f6 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 14:08:39 +0200 Subject: [PATCH 19/20] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e24b2164900..980dba6fb2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ - Maps `unknown_error` span status to `internal_error`. ([#5202](https://github.com/getsentry/relay/pull/5202)) - Add `otp` and `two[-_]factor` to default scrubbing rules. ([#5250](https://github.com/getsentry/relay/pull/5250)) - Add event merging logic for Playstation crashes. ([#5228](https://github.com/getsentry/relay/pull/5228)) +- Implement PII scrubbing for V2 spans. ([#5168](https://github.com/getsentry/relay/pull/5168)) **Bug Fixes**: From f2492c0633f2b1a503f3b69febf7c9fc63f63dce Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Wed, 15 Oct 2025 14:15:54 +0200 Subject: [PATCH 20/20] Formatting --- tests/integration/test_ourlogs.py | 4 ++-- tests/integration/test_spansv2.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_ourlogs.py b/tests/integration/test_ourlogs.py index 62af42b96ba..dd12949260f 100644 --- a/tests/integration/test_ourlogs.py +++ b/tests/integration/test_ourlogs.py @@ -348,7 +348,7 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "trace_id": "5b8efff798038103d269b633813fc60c", "span_id": "eee19b7ec3c1b174", "attributes": { - "test_pii": { "type": "string", "value": expected_scrubbed}, + "test_pii": {"type": "string", "value": expected_scrubbed}, "sentry.browser.name": {"type": "string", "value": "Python Requests"}, "sentry.browser.version": {"type": "string", "value": "2.32"}, "sentry.observed_timestamp_nanos": { @@ -363,7 +363,7 @@ def test_ourlog_extraction_with_string_pii_scrubbing( "value": { "": { "len": mock.ANY, - "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]] + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]], } } } diff --git a/tests/integration/test_spansv2.py b/tests/integration/test_spansv2.py index c79af9b29cf..e1db57756f5 100644 --- a/tests/integration/test_spansv2.py +++ b/tests/integration/test_spansv2.py @@ -652,7 +652,7 @@ def test_spanv2_with_string_pii_scrubbing( "trace_id": "5b8efff798038103d269b633813fc60c", "span_id": "eee19b7ec3c1b174", "attributes": { - "test_pii": { "type": "string", "value": expected_scrubbed}, + "test_pii": {"type": "string", "value": expected_scrubbed}, "sentry.browser.name": {"type": "string", "value": "Python Requests"}, "sentry.browser.version": {"type": "string", "value": "2.32"}, "sentry.observed_timestamp_nanos": { @@ -666,7 +666,7 @@ def test_spanv2_with_string_pii_scrubbing( "value": { "": { "len": mock.ANY, - "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]] + "rem": [[rule_type, mock.ANY, mock.ANY, mock.ANY]], } } }