diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 0cdef6a59..abfd8da37 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -1121,6 +1121,22 @@ The notice is preceded by one blank line (it follows the command's own output, o State lives at `$XDG_CACHE_HOME`|`~/.cache` (Unix/macOS) or `%LOCALAPPDATA%` (Windows) + `/socket-patch/update-check.json` (camelCase JSON: `schemaVersion`, `lastCheckAt`, `latestSeen`, `lastNotifiedAt`; unix seconds). A completed `--update` refreshes `latestSeen`, so the notifier never nags about a version the user just installed. +## Telemetry events + +Each command POSTs its outcome as one JSON event (`{event_sender_created_at, event_type, context: {version, platform, arch, command}, session_id, metadata?, error?: {type, message}}`; no `event_kind`, so the backend stores it as a required event) to `/v0/orgs//telemetry` with the bearer when the run has a token and an org, else anonymously to `/patch/telemetry`. `--no-telemetry`/`SOCKET_TELEMETRY_DISABLED`, `SOCKET_OFFLINE`/`--offline` and `VITEST=true` send nothing. Every event's `context.version` is the client version. Error messages have the home directory replaced by `~`. Every `event_type` must be in depscan's patch event map (`telemetry-discriminators.ts`), or it is stored as `external` and drops out of the patch dashboards; `telemetry.rs` pins that list in a test. + +| Command | Success | Failure | Metadata | +|---|---|---|---| +| `scan` | `patch_scanned` | `patch_scan_failed` | `packages_scanned`, `free_patches`, `paid_patches`, `can_access_paid`, `ecosystems` (found by the crawl), `fallback_to_proxy`, `mode` (requested: `hosted` / `vendored` / `agent`, or `report` with none), `project_modes` (already wired: `hosted` lockfile pins, `vendored` ledger, `agent` manifest) | +| `scan`/`get --mode hosted` (non-empty selection) | `patch_applied` | `patch_apply_failed` | `mode: "hosted"`, `patches_count` (pins confirmed), `files_count` (files rewritten), `dry_run` | +| `apply` (and the nested apply of `scan`/`get --mode agent`) | `patch_applied` | `patch_apply_failed` | `mode: "agent"`, `patches_count`, `dry_run` | +| `vendor` / `scan`/`get --mode vendored` | `patch_vendored` | `patch_vendor_failed` | `patches_count`, `dry_run` | +| `get` | `patch_fetched` | `patch_fetch_failed` | `uuid`, `tier`, `ecosystem`, `download_mode`, `fallback_to_proxy` | +| `remove` / `rollback` | `patch_removed` / `patch_rolled_back` | `patch_remove_failed` / `patch_rollback_failed` | `removed_count` / `rolled_back_count` | +| `list` / `repair` | `patch_listed` / `patch_repaired` | — / `patch_repair_failed` | `patches_count` / `blobs_added`, `blobs_removed`, `bytes_freed` | +| `vex` | `vex_generated` | `vex_failed` | `advisories_count`, `format`, `output_kind` | +| `--update` (an install, not a no-op or `--dry-run`) | `cli_updated` | `cli_update_failed` | `from_version`, `to_version`, `channel`, `pinned`, `forced` / `error_code` (the envelope's) | + ## Environment variables Public configuration uses the `SOCKET_*` names below. The three deprecated v3/v4 environment aliases and `SOCKET_FORCE` were removed in v5; see [Removed env vars](#removed-env-vars). diff --git a/crates/socket-patch-cli/src/commands/get.rs b/crates/socket-patch-cli/src/commands/get.rs index 09a1e0600..e84e63eed 100644 --- a/crates/socket-patch-cli/src/commands/get.rs +++ b/crates/socket-patch-cli/src/commands/get.rs @@ -2210,6 +2210,7 @@ async fn run_get_hosted( None, // `get` is explicit intent: the rollout cap never applies. None, + "get", ) .await } diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 52c1bb2f0..7c5405c86 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -71,11 +71,22 @@ fn emit_json_error( code: &str, message: &str, ) { + emit_scan(&json_error_envelope(common, scan_result, code, message)); +} + +/// The envelope [`emit_json_error`] prints, built without printing it, so +/// [`run_redirect_selected`] can send its failure telemetry first. +fn json_error_envelope( + common: &crate::args::GlobalArgs, + scan_result: Option, + code: &str, + message: &str, +) -> Envelope { let mut env = scan_result.unwrap_or_else(|| scan_envelope(common)); env.mark_error(EnvelopeError::new(code, message)); // The rollout block describes a successful run only. env.extra.remove("rollout"); - emit_scan(&env); + env } /// The hosted run's `redirect` payload: `{mode: "hosted", rewrittenFiles}` @@ -160,10 +171,16 @@ fn refuse( common: &crate::args::GlobalArgs, scan_result: Option, refusal: &socket_patch_core::hosted::engine::Refusal, + json_error: &mut Option, ) -> i32 { eprintln!("Error ({}): {}", refusal.code, refusal.message); if common.json { - emit_json_error(common, scan_result, &refusal.code, &refusal.message); + *json_error = Some(json_error_envelope( + common, + scan_result, + &refusal.code, + &refusal.message, + )); } 1 } @@ -182,6 +199,7 @@ fn refuse( fn acquire_hosted_lock( common: &crate::args::GlobalArgs, scan_result: &mut Option, + json_error: &mut Option, ) -> Result { let socket_dir = common.socket_dir(); let timeout = Duration::from_secs(common.lock_timeout.unwrap_or(0)); @@ -196,7 +214,12 @@ fn acquire_hosted_lock( crate::commands::lock_cli::format_lock_error(&socket_dir, &err, timeout) ); if common.json { - emit_json_error(common, scan_result.take(), code, &message); + *json_error = Some(json_error_envelope( + common, + scan_result.take(), + code, + &message, + )); } Err(1) } @@ -662,6 +685,7 @@ pub(super) async fn run_redirect( scan_result, npm_prior, Some(super::rollout::Gate::new(stage, rows).with_prior(prior)), + "scan", ) .await } @@ -751,8 +775,71 @@ fn discovery_after_writes<'d>( /// would get a bare scan envelope without its own payload). /// `prune_requested` only feeds the `redirect_prune_ignored` warning — /// `get` passes `false`. +/// +/// Telemetry: a run with a non-empty selection reports `patch_applied` +/// (`mode: "hosted"`, sent before the result prints) or, on a non-zero +/// exit before that point, `patch_apply_failed`. `command` names the +/// caller (`scan` / `get`) in the event's context. #[allow(clippy::too_many_arguments)] pub(crate) async fn run_redirect_selected( + common: &crate::args::GlobalArgs, + vex: &crate::commands::vex::VexEmbedArgs, + prune_requested: bool, + api_client: &socket_patch_core::api::client::ApiClient, + selected: &[(String, String)], + scan_result: Option, + npm_prior: Option<&crate::ecosystem_dispatch::NpmCrawlSnapshot>, + rollout: Option>, + command: &'static str, +) -> i32 { + let mut telemetry = HostedTelemetry { + command, + auth: socket_patch_core::telemetry::TelemetryAuth::for_client(api_client), + sent: false, + json_error: None, + }; + let code = run_redirect_selected_untracked( + common, + vex, + prune_requested, + api_client, + selected, + scan_result, + npm_prior, + rollout, + &mut telemetry, + ) + .await; + if code != 0 && !telemetry.sent && !selected.is_empty() { + socket_patch_core::telemetry::track_patch_hosted_failed( + command, + "hosted redirect failed", + common.dry_run, + &telemetry.auth, + ) + .await; + } + // The `--json` error envelope prints only after the failure event went + // out, so a consumer that closes stdout on it cannot drop the event. + if let Some(envelope) = telemetry.json_error.take() { + emit_scan(&envelope); + } + code +} + +/// Where [`run_redirect_selected`]'s outcome event goes, and whether the +/// success event already went out. +struct HostedTelemetry { + command: &'static str, + auth: socket_patch_core::telemetry::TelemetryAuth, + sent: bool, + /// A `--json` error envelope a failure path built, printed by + /// [`run_redirect_selected`] after the failure event is sent. + json_error: Option, +} + +#[allow(clippy::too_many_arguments)] +async fn run_redirect_selected_untracked( common: &crate::args::GlobalArgs, vex: &crate::commands::vex::VexEmbedArgs, prune_requested: bool, @@ -763,6 +850,7 @@ pub(crate) async fn run_redirect_selected( // `scan`'s rollout gate: NEW rows past the budget are deferred after // every write-free eligibility check below (§5.2). `get` passes `None`. mut rollout: Option>, + telemetry: &mut HostedTelemetry, ) -> i32 { use socket_patch_core::hosted::engine::{ self, Candidate, CandidateFiles, RewriteOptions, SkippedPatch, @@ -813,12 +901,12 @@ pub(crate) async fn run_redirect_selected( format_error_line(&message) ); if common.json { - emit_json_error( + telemetry.json_error = Some(json_error_envelope( common, scan_result.take(), "reference_resolve_failed", &message, - ); + )); } return 1; } @@ -853,6 +941,7 @@ pub(crate) async fn run_redirect_selected( common, scan_result.take(), &engine::bun_lockb_symlink_refusal(), + &mut telemetry.json_error, ); } // A workspace member whose lock lives in an ancestor directory (pnpm @@ -867,7 +956,12 @@ pub(crate) async fn run_redirect_selected( ) .await { - return refuse(common, scan_result.take(), &refusal); + return refuse( + common, + scan_result.take(), + &refusal, + &mut telemetry.json_error, + ); } // vlt artifact preflight: before any takeover or rewrite (dry runs @@ -928,7 +1022,7 @@ pub(crate) async fn run_redirect_selected( }) }); let mut lock: Option = if !common.dry_run && !candidates.is_empty() && may_write { - match acquire_hosted_lock(common, &mut scan_result) { + match acquire_hosted_lock(common, &mut scan_result, &mut telemetry.json_error) { Ok(guard) => Some(guard), Err(code) => return code, } @@ -972,7 +1066,14 @@ pub(crate) async fn run_redirect_selected( .await { Ok(t) => t, - Err(refusal) => return refuse(common, scan_result.take(), &refusal), + Err(refusal) => { + return refuse( + common, + scan_result.take(), + &refusal, + &mut telemetry.json_error, + ) + } }; takeover .stage(common, &group, &mut candidates, &mut skipped) @@ -1305,7 +1406,7 @@ pub(crate) async fn run_redirect_selected( } = takeover.finish(common.dry_run, &mut vendor_state, &done.rewrite.warnings); // A takeover writes the vendored ledger: hold the lock for it too. if lock.is_none() && !common.dry_run && !takeover_migrated.is_empty() { - match acquire_hosted_lock(common, &mut scan_result) { + match acquire_hosted_lock(common, &mut scan_result, &mut telemetry.json_error) { Ok(guard) => lock = Some(guard), Err(code) => return code, } @@ -1328,13 +1429,23 @@ pub(crate) async fn run_redirect_selected( // hosted → upstream restore's staged flush) already refuses linked // files, so the write side must too. if let Some(refusal) = engine::guard(&view, &done, &candidates) { - return refuse(common, scan_result.take(), &refusal); + return refuse( + common, + scan_result.take(), + &refusal, + &mut telemetry.json_error, + ); } // Defense in depth for the Gradle planner: a settings file it plans to // CREATE (it never read one) must not already be on disk, or the // atomic write would replace the user's settings with the apply line. if let Some(refusal) = created_settings_over_existing(&common.cwd, &done) { - return refuse(common, scan_result.take(), &refusal); + return refuse( + common, + scan_result.take(), + &refusal, + &mut telemetry.json_error, + ); } if !common.dry_run { @@ -1416,12 +1527,12 @@ pub(crate) async fn run_redirect_selected( { eprintln!("{}", format_error_line(&message)); if common.json { - emit_json_error( + telemetry.json_error = Some(json_error_envelope( common, scan_result.take(), "lockfile_write_failed", &message, - ); + )); } return 1; } @@ -1724,6 +1835,19 @@ pub(crate) async fn run_redirect_selected( &confirmed, &skipped, ); + // Sent before the result prints: a consumer that closes stdout early + // (SIGPIPE on the first write) must not lose it. + if !selected.is_empty() { + socket_patch_core::telemetry::track_patch_hosted( + telemetry.command, + confirmed.len(), + done.rewritten.len(), + common.dry_run, + &telemetry.auth, + ) + .await; + } + telemetry.sent = true; if common.json { // Record the redirect outcome into the caller's envelope (scan's // discovery payload, or get's narrowing events): per-patch events @@ -2346,6 +2470,7 @@ pub(crate) fn boxed_run_redirect_selected<'a>( scan_result: Option, npm_prior: Option<&'a crate::ecosystem_dispatch::NpmCrawlSnapshot>, rollout: Option>, + command: &'static str, ) -> std::pin::Pin + 'a>> { Box::pin(run_redirect_selected( common, @@ -2356,6 +2481,7 @@ pub(crate) fn boxed_run_redirect_selected<'a>( scan_result, npm_prior, rollout, + command, )) } diff --git a/crates/socket-patch-cli/src/commands/scan/mod.rs b/crates/socket-patch-cli/src/commands/scan/mod.rs index 6d204040e..04934c089 100644 --- a/crates/socket-patch-cli/src/commands/scan/mod.rs +++ b/crates/socket-patch-cli/src/commands/scan/mod.rs @@ -1510,6 +1510,17 @@ fn emit_scan_error(common: &GlobalArgs, error: EnvelopeError) { emit_scan(&env); } +/// The ecosystems `purls` span, by CLI name, sorted and deduplicated. +fn scanned_ecosystems(purls: &[String]) -> Vec { + purls + .iter() + .filter_map(|purl| Ecosystem::from_purl(purl)) + .map(|eco| eco.cli_name().to_string()) + .collect::>() + .into_iter() + .collect() +} + pub async fn run(args: ScanArgs) -> i32 { // Scan's telemetry sends run off the critical path: each is spawned // where its event fires and flushed before the first stdout write that @@ -2050,6 +2061,23 @@ async fn run_scan( read_set: read_set.as_ref(), }) }; + // Telemetry: the mode this run was asked for, and the modes the + // project is already wired for (it may hold more than one mid-migration). + let requested_mode = args.mode.map_or("report", ScanMode::cli_name); + let project_modes: Vec<&str> = [ + ( + !hosted_pin_list.is_empty() || !hosted_unlocked_pins.is_empty(), + "hosted", + ), + (!vendored_purls.is_empty(), "vendored"), + ( + existing_manifest.is_some_and(|m| !m.patches.is_empty()), + "agent", + ), + ] + .into_iter() + .filter_map(|(present, mode)| present.then_some(mode)) + .collect(); let hosted_state = (!args.common.is_global()) .then(|| crate::commands::hosted_state_from_pins(&hosted_pin_list)); let redirect_state = hosted_state.as_ref(); @@ -2233,12 +2261,10 @@ async fn run_scan( 0, 0, false, - args.common - .ecosystems - .clone() - .unwrap_or_default() - .as_slice(), + &[], false, + requested_mode, + &project_modes, &telemetry_auth, ); // The result prints right away: nothing to overlap the send with. @@ -2536,19 +2562,19 @@ async fn run_scan( } let total_patches = free_patches + paid_patches; - // Telemetry: record the scan outcome with the per-tier counts. + // Telemetry: record the scan outcome with the per-tier counts and the + // ecosystems the crawl actually found (not the `--ecosystems` filter, + // which is empty on a default run). spawn_patch_scanned( telemetry, package_count, free_patches, paid_patches, can_access_paid_patches, - args.common - .ecosystems - .clone() - .unwrap_or_default() - .as_slice(), + &scanned_ecosystems(&all_purls), fallback_to_proxy, + requested_mode, + &project_modes, &telemetry_auth, ); @@ -3121,6 +3147,7 @@ async fn run_scan( None, npm_crawl.as_ref(), Some(rollout::Gate::new(&mut stage, rows).with_prior(prior_discovery)), + "scan", ) .await; } diff --git a/crates/socket-patch-cli/src/commands/update.rs b/crates/socket-patch-cli/src/commands/update.rs index 9fd8fef5e..93a5c85d6 100644 --- a/crates/socket-patch-cli/src/commands/update.rs +++ b/crates/socket-patch-cli/src/commands/update.rs @@ -8,6 +8,7 @@ //! machinery lives in `socket_patch_core::update`. use clap::Args; +use socket_patch_core::telemetry::{track_cli_update_failed, track_cli_updated}; use socket_patch_core::update::{ self as core_update, asset_name_for_target, channel_label_for, current_version, detect_channel, fetch_latest_version, is_newer, upgrade_hint_for, ChannelEnv, InstallChannel, UpdateEndpoints, @@ -75,8 +76,11 @@ fn parse_version_pin(raw: &str) -> Result { /// Emit an error in the mode-appropriate shape and return the exit code. /// The envelope keeps the message verbatim; the human line capitalizes it -/// (`Error: Could not check for updates: ...`). -fn fail(args: &UpdateArgs, code: &str, message: &str) -> i32 { +/// (`Error: Could not check for updates: ...`). The `cli_update_failed` +/// telemetry event goes out first, so a closed stdout cannot lose it (and +/// `--offline` disables it like every other send). +async fn fail(args: &UpdateArgs, code: &str, message: &str) -> i32 { + track_cli_update_failed(code, message, &args.common.telemetry_auth()).await; if args.common.json { let env = error_envelope(Command::Update, args.common.dry_run, code, message); println!("{}", env.to_pretty_json()); @@ -86,6 +90,19 @@ fn fail(args: &UpdateArgs, code: &str, message: &str) -> i32 { 1 } +/// A stable machine name for `channel` (telemetry), unlike the +/// human-facing [`channel_label`] (`cargo install`, `the RubyGems launcher`). +fn channel_id(channel: InstallChannel) -> &'static str { + match channel { + InstallChannel::Standalone => "standalone", + InstallChannel::Npm => "npm", + InstallChannel::Pypi => "pypi", + InstallChannel::Cargo => "cargo", + InstallChannel::LauncherCache => "launcher_cache", + InstallChannel::Homebrew => "homebrew", + } +} + /// The no-op message when there is nothing to install: a pin already /// satisfied, the latest release already running, or a build newer than /// the latest release (`latest` never downgrades). @@ -193,13 +210,14 @@ pub async fn run(args: UpdateArgs) -> i32 { "offline", "update requires network access to check releases and cannot run with \ --offline/SOCKET_OFFLINE (strict airgap)", - ); + ) + .await; } // 2. Where is this binary, and who manages it? Zero network so far. let install_path = match core_update::resolve_install_path() { Ok(p) => p, - Err(e) => return fail(&args, e.error_code(), &e.to_string()), + Err(e) => return fail(&args, e.error_code(), &e.to_string()).await, }; let channel = detect_channel(&install_path, &ChannelEnv::from_env()); let hint = upgrade_hint_for(channel, &install_path); @@ -226,7 +244,8 @@ pub async fn run(args: UpdateArgs) -> i32 { channel_label_for(channel, &install_path), hint ), - ); + ) + .await; } } @@ -239,7 +258,9 @@ pub async fn run(args: UpdateArgs) -> i32 { Ok(v) => (v, true), // Unreachable via clap (value_parser validates), but the env // path deserves a real error over a panic. - Err(e) => return fail(&args, "check_failed", &format!("invalid version pin: {e}")), + Err(e) => { + return fail(&args, "check_failed", &format!("invalid version pin: {e}")).await + } }, None => { // The check can take a while on a slow network (two probes, @@ -251,7 +272,7 @@ pub async fn run(args: UpdateArgs) -> i32 { status.finish(); match latest { Ok(v) => (v, false), - Err(e) => return fail(&args, e.error_code(), &e.to_string()), + Err(e) => return fail(&args, e.error_code(), &e.to_string()).await, } } }; @@ -367,7 +388,7 @@ pub async fn run(args: UpdateArgs) -> i32 { or re-run the installer", ); } - return fail(&args, e.error_code(), &message); + return fail(&args, e.error_code(), &message).await; } }; @@ -375,6 +396,16 @@ pub async fn run(args: UpdateArgs) -> i32 { note_warning(&mut warnings, quiet, "update_warning", warning.clone()); } + // Before the result prints, like `fail`'s failure event. + track_cli_updated( + &target_version.to_string(), + channel_id(channel), + pinned, + args.force, + &args.common.telemetry_auth(), + ) + .await; + if args.common.json { let mut env = Envelope::new(Command::Update); env.record( diff --git a/crates/socket-patch-cli/tests/cli/telemetry_e2e.rs b/crates/socket-patch-cli/tests/cli/telemetry_e2e.rs index 639278b80..3116c3985 100644 --- a/crates/socket-patch-cli/tests/cli/telemetry_e2e.rs +++ b/crates/socket-patch-cli/tests/cli/telemetry_e2e.rs @@ -1069,3 +1069,350 @@ async fn scan_delivers_telemetry_before_writing_to_a_closed_stderr() { ); } } + +// --------------------------------------------------------------------------- +// hosted mode (the v5 `scan` default) +// --------------------------------------------------------------------------- + +const HOSTED_NAME: &str = "telemetry-hosted"; +const HOSTED_PURL: &str = "pkg:npm/telemetry-hosted@1.0.0"; +const HOSTED_UUID: &str = "11111111-1111-4111-8111-111111111111"; +const HOSTED_URL: &str = "https://patch.socket.dev/patch/npm/telemetry-hosted/1.0.0/22222222-2222-4222-8222-222222222222/11111111-1111-4111-8111-111111111111/telemetry-hosted-1.0.0.tgz"; + +/// An npm project whose lockfile resolves [`HOSTED_NAME`] upstream (the +/// `covgap_commands_scan_hosted.rs` `write_npm_project` shape). +fn write_hosted_npm_project(root: &Path) { + std::fs::write( + root.join("package.json"), + format!( + r#"{{ "name": "consumer", "version": "0.0.0", "dependencies": {{ "{HOSTED_NAME}": "1.0.0" }} }}"# + ), + ) + .unwrap(); + write_npm_package(root, HOSTED_NAME, "1.0.0"); + std::fs::write( + root.join("package-lock.json"), + format!( + r#"{{ + "name": "consumer", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": {{ + "": {{ "name": "consumer", "version": "0.0.0", "dependencies": {{ "{HOSTED_NAME}": "1.0.0" }} }}, + "node_modules/{HOSTED_NAME}": {{ + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/{HOSTED_NAME}/-/{HOSTED_NAME}-1.0.0.tgz", + "integrity": "sha512-UPSTREAMupstream==" + }} + }} +}} +"# + ), + ) + .unwrap(); +} + +/// The API surface a hosted scan of [`write_hosted_npm_project`] reads: +/// batch discovery, the per-package search, the reference grant (answered +/// with `reference_status`) and the patch record. Telemetry answers 201. +async fn setup_hosted_mock(reference_status: u16) -> MockServer { + let batch = serde_json::json!({ + "packages": [{ + "purl": HOSTED_PURL, + "patches": [{ + "uuid": HOSTED_UUID, "purl": HOSTED_PURL, "tier": "free", + "cveIds": [], "ghsaIds": [], "severity": "high", + "title": "telemetry hosted fixture" + }] + }], + "canAccessPaidPatches": false, + }); + let mock = setup_mock(batch, None).await; + Mock::given(method("GET")) + .and(wiremock::matchers::path_regex(format!( + "^/v0/orgs/{ORG_SLUG}/patches/by-package/.+$" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "patches": [{ + "uuid": HOSTED_UUID, "purl": HOSTED_PURL, + "publishedAt": "2024-01-01T00:00:00Z", + "description": "x", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(&mock) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG_SLUG}/patches/package"))) + .respond_with( + ResponseTemplate::new(reference_status).set_body_json(serde_json::json!({ + "results": { + HOSTED_UUID: { + "status": "granted", + "url": HOSTED_URL, + "purl": HOSTED_PURL, + "artifacts": [{ + "kind": "tarball", + "url": HOSTED_URL, + "integrity": { "sha512": "sha512-PATCHEDpatchedPATCHEDpatched0123456789==" } + }], + "registryOverride": null + } + } + })), + ) + .mount(&mock) + .await; + Mock::given(method("GET")) + .and(path(format!( + "/v0/orgs/{ORG_SLUG}/patches/view/{HOSTED_UUID}" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "uuid": HOSTED_UUID, + "purl": HOSTED_PURL, + "publishedAt": "2024-01-01T00:00:00Z", + "files": {}, + "vulnerabilities": {}, + "description": "x", "license": "MIT", "tier": "free" + }))) + .mount(&mock) + .await; + mock +} + +/// Every telemetry body the mock received, in order. +async fn telemetry_bodies(mock: &MockServer) -> Vec { + mock.received_requests() + .await + .expect("wiremock allows recording") + .iter() + .filter(|req| { + req.url + .path() + .ends_with(&format!("/v0/orgs/{ORG_SLUG}/telemetry")) + }) + .map(|req| serde_json::from_slice(&req.body).expect("telemetry body is JSON")) + .collect() +} + +/// v5's default `scan` pins patched dependencies to the hosted patch +/// server. That IS the patch adoption, so besides `patch_scanned` (with +/// the ecosystems the crawl found) it reports `patch_applied` tagged +/// `mode: "hosted"` — the event the backend's adoption views count. +#[tokio::test] +async fn hosted_scan_reports_patch_applied_with_hosted_mode() { + let mock = setup_hosted_mock(200).await; + let tmp = tempfile::tempdir().expect("tempdir"); + write_hosted_npm_project(tmp.path()); + + let (code, stdout, stderr) = run_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]); + assert_eq!(code, 0, "stdout:\n{stdout}\nstderr:\n{stderr}"); + let lock = std::fs::read_to_string(tmp.path().join("package-lock.json")).unwrap(); + assert!(lock.contains(HOSTED_URL), "the lockfile is pinned: {lock}"); + let doc: serde_json::Value = serde_json::from_str(&stdout).expect("JSON envelope"); + + let bodies = telemetry_bodies(&mock).await; + let types: Vec<&str> = bodies + .iter() + .map(|b| b["event_type"].as_str().unwrap()) + .collect(); + assert_eq!(types, ["patch_scanned", "patch_applied"], "{bodies:#?}"); + + let scanned = &bodies[0]; + assert_eq!( + scanned["metadata"]["ecosystems"], + serde_json::json!(["npm"]) + ); + // Default mode, on a project nothing is wired into yet. + assert_eq!(scanned["metadata"]["mode"], "hosted"); + assert_eq!(scanned["metadata"]["project_modes"], serde_json::json!([])); + assert_eq!(scanned["metadata"]["free_patches"], 1); + + let applied = &bodies[1]; + assert_eq!(applied["context"]["command"], "scan"); + assert_eq!(applied["metadata"]["mode"], "hosted"); + // v5.0 records each confirmed pin as an `applied` event tagged + // `details.mode: "hosted"` (v4's `redirect.redirected` count). + let hosted_applied = doc["events"] + .as_array() + .into_iter() + .flatten() + .filter(|e| e["details"]["mode"] == "hosted" && e["action"] == "applied") + .count(); + assert_eq!( + applied["metadata"]["patches_count"], hosted_applied, + "patches_count is the envelope's hosted applied events: {doc:#}" + ); + assert_eq!(applied["metadata"]["patches_count"], 1); + assert_eq!( + applied["metadata"]["files_count"], + doc["redirect"]["rewrittenFiles"].as_array().unwrap().len(), + "files_count is the envelope's rewrittenFiles: {doc:#}" + ); + assert_eq!(applied["metadata"]["dry_run"], false); + assert_eq!( + applied["session_id"], scanned["session_id"], + "one run, one session" + ); + + // A re-scan sees the hosted pins the first run wrote. + let (code, stdout, stderr) = run_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]); + assert_eq!(code, 0, "stdout:\n{stdout}\nstderr:\n{stderr}"); + let bodies = telemetry_bodies(&mock).await; + let rescanned = bodies + .iter() + .filter(|b| b["event_type"] == "patch_scanned") + .nth(1) + .expect("the re-scan's patch_scanned"); + assert_eq!( + rescanned["metadata"]["project_modes"], + serde_json::json!(["hosted"]) + ); + assert_ne!(rescanned["session_id"], scanned["session_id"]); +} + +/// `patch_scanned` carries the requested mode and the modes the project is +/// already wired for: here an agent manifest, scanned with `--mode agent`. +#[tokio::test] +async fn scan_reports_requested_and_project_modes() { + let mock = setup_mock( + serde_json::json!({ "packages": [], "canAccessPaidPatches": false }), + None, + ) + .await; + let tmp = tempfile::tempdir().expect("tempdir"); + write_root_package_json(tmp.path()); + write_npm_package(tmp.path(), "minimist", "1.2.2"); + let socket = tmp.path().join(".socket"); + std::fs::create_dir_all(&socket).unwrap(); + std::fs::write( + socket.join("manifest.json"), + serde_json::json!({ + "patches": { + "pkg:npm/minimist@1.2.2": { + "uuid": "33333333-3333-4333-8333-333333333333", + "exportedAt": "2024-01-01T00:00:00Z", + "files": {}, + "vulnerabilities": {}, + "description": "x", + "license": "MIT", + "tier": "free" + } + } + }) + .to_string(), + ) + .unwrap(); + + let (code, stdout, stderr) = run_cmd( + tmp.path(), + &mock.uri(), + "scan", + &["--mode", "agent", "--dry-run"], + &[], + ); + assert_eq!(code, 0, "stdout:\n{stdout}\nstderr:\n{stderr}"); + + let bodies = telemetry_bodies(&mock).await; + let scanned = bodies + .iter() + .find(|b| b["event_type"] == "patch_scanned") + .unwrap_or_else(|| panic!("no patch_scanned: {bodies:#?}")); + assert_eq!(scanned["metadata"]["mode"], "agent"); + assert_eq!( + scanned["metadata"]["project_modes"], + serde_json::json!(["agent"]) + ); + assert_eq!( + scanned["context"]["version"], + env!("CARGO_PKG_VERSION"), + "every event carries the client version" + ); +} + +/// A hosted run that fails after selecting patches reports +/// `patch_apply_failed` (`mode: "hosted"`) and leaves the lockfile alone. +#[tokio::test] +async fn hosted_scan_failure_reports_patch_apply_failed() { + let mock = setup_hosted_mock(500).await; + let tmp = tempfile::tempdir().expect("tempdir"); + write_hosted_npm_project(tmp.path()); + + let (code, stdout, stderr) = run_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]); + assert_eq!(code, 1, "stdout:\n{stdout}\nstderr:\n{stderr}"); + + let bodies = telemetry_bodies(&mock).await; + let types: Vec<&str> = bodies + .iter() + .map(|b| b["event_type"].as_str().unwrap()) + .collect(); + assert_eq!( + types, + ["patch_scanned", "patch_apply_failed"], + "{bodies:#?}" + ); + let failed = &bodies[1]; + assert_eq!(failed["context"]["command"], "scan"); + assert_eq!(failed["metadata"]["mode"], "hosted"); + assert_eq!(failed["error"]["type"], "Error"); +} + +/// The `--json` failure envelope (every run here is `--json`) prints only +/// after `patch_apply_failed` is sent: with stdout closed from the start, +/// its first write raises SIGPIPE, and the event must already be out. +#[tokio::test] +async fn hosted_scan_failure_delivers_telemetry_before_a_closed_stdout() { + let mock = setup_hosted_mock(500).await; + let tmp = tempfile::tempdir().expect("tempdir"); + write_hosted_npm_project(tmp.path()); + + let mut child = build_cmd(tmp.path(), &mock.uri(), "scan", &[], &[]) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn socket-patch"); + drop(child.stdout.take()); + let status = child.wait().expect("wait socket-patch"); + assert!(!status.success(), "{status:?}"); + + let bodies = telemetry_bodies(&mock).await; + let types: Vec<&str> = bodies + .iter() + .map(|b| b["event_type"].as_str().unwrap()) + .collect(); + assert_eq!( + types, + ["patch_scanned", "patch_apply_failed"], + "{bodies:#?}" + ); +} + +/// `get --mode hosted` reports through the same path, under its own +/// command name. +#[tokio::test] +async fn hosted_get_reports_patch_applied_under_get() { + let mock = setup_hosted_mock(200).await; + let tmp = tempfile::tempdir().expect("tempdir"); + write_hosted_npm_project(tmp.path()); + + let (code, stdout, stderr) = run_cmd( + tmp.path(), + &mock.uri(), + "get", + &[HOSTED_PURL, "--mode", "hosted", "--yes"], + &[], + ); + assert_eq!(code, 0, "stdout:\n{stdout}\nstderr:\n{stderr}"); + + let bodies = telemetry_bodies(&mock).await; + let applied: Vec<&serde_json::Value> = bodies + .iter() + .filter(|b| b["event_type"] == "patch_applied") + .collect(); + assert_eq!(applied.len(), 1, "{bodies:#?}"); + assert_eq!(applied[0]["context"]["command"], "get"); + assert_eq!(applied[0]["metadata"]["mode"], "hosted"); + assert_eq!(applied[0]["metadata"]["patches_count"], 1); +} diff --git a/crates/socket-patch-cli/tests/update/main.rs b/crates/socket-patch-cli/tests/update/main.rs index 0b4629b5b..69da0e5f2 100644 --- a/crates/socket-patch-cli/tests/update/main.rs +++ b/crates/socket-patch-cli/tests/update/main.rs @@ -13,4 +13,5 @@ mod covgap_commands_update; mod covgap_update_download; mod covgap_update_swap; mod self_update_channels_e2e; +mod self_update_telemetry_e2e; mod update_notifier_e2e; diff --git a/crates/socket-patch-cli/tests/update/self_update_telemetry_e2e.rs b/crates/socket-patch-cli/tests/update/self_update_telemetry_e2e.rs new file mode 100644 index 000000000..350d28c65 --- /dev/null +++ b/crates/socket-patch-cli/tests/update/self_update_telemetry_e2e.rs @@ -0,0 +1,141 @@ +//! `socket-patch --update` reports its outcome to the telemetry endpoint: +//! `cli_updated` after a swap, `cli_update_failed` (with the envelope's +//! error code) on a failure. Each run drives a staged COPY of the binary +//! (`update_fixture::staged_install`) and points the anonymous proxy +//! telemetry route at a local wiremock; the workspace-wide +//! `SOCKET_TELEMETRY_DISABLED=1` (`.cargo/config.toml`) is overridden for +//! the child only. + +use crate::update_fixture; + +use update_fixture::{make_served_binary, run_installed, staged_install, FakeReleaseBuilder}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +const CURRENT: &str = env!("CARGO_PKG_VERSION"); + +/// A telemetry recorder on the public proxy's `/patch/telemetry` route. +async fn telemetry_mock() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/patch/telemetry")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + server +} + +/// The telemetry bodies `server` received, in order. +async fn telemetry_events(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap() + .iter() + .map(|req| { + assert!( + req.headers.get("authorization").is_none(), + "the anonymous proxy route never carries a bearer" + ); + serde_json::from_slice(&req.body).expect("telemetry body is JSON") + }) + .collect() +} + +#[tokio::test] +async fn successful_update_reports_cli_updated() { + let install = staged_install(); + let (served, _) = make_served_binary(); + let release = FakeReleaseBuilder::new(CURRENT) + .asset_for_current_target(&served) + .mount() + .await; + let telemetry = telemetry_mock().await; + + let (code, stdout, stderr) = run_installed( + &install, + &["--update", "--force", "--yes"], + &[ + ("SOCKET_UPDATE_BASE_URL", &release.base_url), + ("SOCKET_PROXY_URL", &telemetry.uri()), + ("SOCKET_TELEMETRY_DISABLED", "0"), + ], + ); + assert_eq!(code, 0, "stdout:\n{stdout}\nstderr:\n{stderr}"); + + let events = telemetry_events(&telemetry).await; + assert_eq!(events.len(), 1, "exactly one event: {events:#?}"); + let event = &events[0]; + assert_eq!(event["event_type"], "cli_updated"); + assert_eq!(event["context"]["command"], "update"); + assert_eq!(event["context"]["version"], CURRENT); + assert_eq!(event["metadata"]["from_version"], CURRENT); + assert_eq!(event["metadata"]["to_version"], CURRENT); + assert_eq!(event["metadata"]["channel"], "standalone"); + assert_eq!(event["metadata"]["forced"], true); + assert_eq!(event["metadata"]["pinned"], false); + assert!(event.get("error").is_none()); + + release.verify_request_hygiene().await; +} + +#[tokio::test] +async fn failed_update_reports_cli_update_failed_with_its_code() { + let install = staged_install(); + let (served, _) = make_served_binary(); + let release = FakeReleaseBuilder::new("9.9.9") + .asset_for_current_target(&served) + .omit_sums_file() + .expect_asset_downloads(0) + .mount() + .await; + let telemetry = telemetry_mock().await; + + let (code, stdout, stderr) = run_installed( + &install, + &["--update", "--yes", "--json"], + &[ + ("SOCKET_UPDATE_BASE_URL", &release.base_url), + ("SOCKET_PROXY_URL", &telemetry.uri()), + ("SOCKET_TELEMETRY_DISABLED", "0"), + ], + ); + assert_eq!(code, 1, "stdout:\n{stdout}\nstderr:\n{stderr}"); + let envelope: serde_json::Value = serde_json::from_str(&stdout).expect("JSON envelope"); + let envelope_code = envelope["error"]["code"] + .as_str() + .expect("the failure envelope carries a code"); + + let events = telemetry_events(&telemetry).await; + assert_eq!(events.len(), 1, "exactly one event: {events:#?}"); + let event = &events[0]; + assert_eq!(event["event_type"], "cli_update_failed"); + assert_eq!(event["context"]["command"], "update"); + assert_eq!(event["metadata"]["error_code"], envelope_code); + assert!( + event["error"]["message"] + .as_str() + .is_some_and(|m| m.contains("SHA256SUMS")), + "the error message rides along: {event:#}" + ); + + install.assert_binary_intact(); +} + +/// `--offline` refuses before any network, telemetry included. +#[tokio::test] +async fn offline_update_sends_nothing() { + let install = staged_install(); + let telemetry = telemetry_mock().await; + + let (code, stdout, stderr) = run_installed( + &install, + &["--update", "--yes", "--offline"], + &[ + ("SOCKET_PROXY_URL", &telemetry.uri()), + ("SOCKET_TELEMETRY_DISABLED", "0"), + ], + ); + assert_eq!(code, 1, "stdout:\n{stdout}\nstderr:\n{stderr}"); + assert!(telemetry_events(&telemetry).await.is_empty()); +} diff --git a/crates/socket-patch-core/src/telemetry.rs b/crates/socket-patch-core/src/telemetry.rs index ce0b2f8de..8071ef6fc 100644 --- a/crates/socket-patch-core/src/telemetry.rs +++ b/crates/socket-patch-core/src/telemetry.rs @@ -52,9 +52,36 @@ enum PatchTelemetryEventType { // OpenVEX attestation (added in #81) VexGenerated, VexFailed, + // Self-update (`socket-patch --update`) + CliUpdated, + CliUpdateFailed, } impl PatchTelemetryEventType { + /// Every variant, for the backend-contract test. + #[cfg(test)] + const ALL: [Self; 19] = [ + Self::PatchApplied, + Self::PatchApplyFailed, + Self::PatchRemoved, + Self::PatchRemoveFailed, + Self::PatchRolledBack, + Self::PatchRollbackFailed, + Self::PatchScanned, + Self::PatchScanFailed, + Self::PatchFetched, + Self::PatchFetchFailed, + Self::PatchVendored, + Self::PatchVendorFailed, + Self::PatchListed, + Self::PatchRepaired, + Self::PatchRepairFailed, + Self::VexGenerated, + Self::VexFailed, + Self::CliUpdated, + Self::CliUpdateFailed, + ]; + /// Return the wire-format string for this event type. fn as_str(&self) -> &'static str { match self { @@ -75,6 +102,8 @@ impl PatchTelemetryEventType { Self::PatchRepairFailed => "patch_repair_failed", Self::VexGenerated => "vex_generated", Self::VexFailed => "vex_failed", + Self::CliUpdated => "cli_updated", + Self::CliUpdateFailed => "cli_update_failed", } } } @@ -466,12 +495,16 @@ async fn fire_prepared(prepared: Option) { } } -/// Track a successful patch application. +/// Track a successful in-place (agent mode) patch application. pub async fn track_patch_applied(patches_count: usize, dry_run: bool, auth: &TelemetryAuth) { fire( PatchTelemetryEventType::PatchApplied, "apply", - serde_json::json!({ "patches_count": patches_count, "dry_run": dry_run }), + serde_json::json!({ + "patches_count": patches_count, + "dry_run": dry_run, + "mode": "agent", + }), None::<&str>, auth, ) @@ -490,7 +523,52 @@ pub async fn track_patch_apply_failed( fire( PatchTelemetryEventType::PatchApplyFailed, "apply", - serde_json::json!({ "dry_run": dry_run }), + serde_json::json!({ "dry_run": dry_run, "mode": "agent" }), + Some(error), + auth, + ) + .await; +} + +/// Track a hosted-mode run (`scan`/`get --mode hosted`) that pinned +/// patched dependencies to Socket's hosted patch server. Sent as +/// `patch_applied` with `mode: "hosted"`: the lockfile pin is how hosted +/// mode applies a patch, and the backend's adoption views count +/// `patch_applied`. `files_count` is the number of lockfiles/manifests +/// rewritten. +pub async fn track_patch_hosted( + command: &'static str, + patches_count: usize, + files_count: usize, + dry_run: bool, + auth: &TelemetryAuth, +) { + fire( + PatchTelemetryEventType::PatchApplied, + command, + serde_json::json!({ + "patches_count": patches_count, + "files_count": files_count, + "dry_run": dry_run, + "mode": "hosted", + }), + None::<&str>, + auth, + ) + .await; +} + +/// Track a failed hosted-mode run (see [`track_patch_hosted`]). +pub async fn track_patch_hosted_failed( + command: &'static str, + error: impl std::fmt::Display, + dry_run: bool, + auth: &TelemetryAuth, +) { + fire( + PatchTelemetryEventType::PatchApplyFailed, + command, + serde_json::json!({ "dry_run": dry_run, "mode": "hosted" }), Some(error), auth, ) @@ -578,9 +656,12 @@ pub async fn track_patch_rollback_failed(error: impl std::fmt::Display, auth: &T // --------------------------------------------------------------------------- /// The whole `patch_scanned` event — type, command and metadata (per-tier -/// patch counts and whether the call was downgraded to the public proxy -/// after an auth-endpoint 401/403). Both wrappers below build their event -/// here, so the inline and background paths can never drift apart. +/// patch counts, whether the call was downgraded to the public proxy +/// after an auth-endpoint 401/403, the mode the run was asked for — +/// `hosted` / `vendored` / `agent`, or `report` with none — and the modes +/// the project is already wired for, from its hosted lockfile pins, vendor +/// ledger and agent manifest). Both wrappers below build their event here, +/// so the inline and background paths can never drift apart. #[allow(clippy::too_many_arguments)] fn prepare_patch_scanned( packages_scanned: usize, @@ -589,6 +670,8 @@ fn prepare_patch_scanned( can_access_paid: bool, ecosystems: &[String], fallback_to_proxy: bool, + mode: &str, + project_modes: &[&str], auth: &TelemetryAuth, ) -> Option { prepare( @@ -601,6 +684,8 @@ fn prepare_patch_scanned( "can_access_paid": can_access_paid, "ecosystems": ecosystems, "fallback_to_proxy": fallback_to_proxy, + "mode": mode, + "project_modes": project_modes, }), None::<&str>, auth, @@ -628,6 +713,8 @@ pub async fn track_patch_scanned( can_access_paid: bool, ecosystems: &[String], fallback_to_proxy: bool, + mode: &str, + project_modes: &[&str], auth: &TelemetryAuth, ) { fire_prepared(prepare_patch_scanned( @@ -637,6 +724,8 @@ pub async fn track_patch_scanned( can_access_paid, ecosystems, fallback_to_proxy, + mode, + project_modes, auth, )) .await; @@ -654,6 +743,8 @@ pub fn spawn_patch_scanned( can_access_paid: bool, ecosystems: &[String], fallback_to_proxy: bool, + mode: &str, + project_modes: &[&str], auth: &TelemetryAuth, ) { pending.spawn_prepared(prepare_patch_scanned( @@ -663,6 +754,8 @@ pub fn spawn_patch_scanned( can_access_paid, ecosystems, fallback_to_proxy, + mode, + project_modes, auth, )); } @@ -852,6 +945,54 @@ pub async fn track_vex_failed(error: impl std::fmt::Display, auth: &TelemetryAut .await; } +// --------------------------------------------------------------------------- +// Self-update trackers +// --------------------------------------------------------------------------- + +/// Track a successful self-update (`socket-patch --update`). The event's +/// `context.version` is the running (pre-update) build; `to_version` is +/// what was installed. `channel` is the detected install channel +/// (`standalone`, or the package manager a `--force` overrode). +pub async fn track_cli_updated( + to_version: &str, + channel: &str, + pinned: bool, + forced: bool, + auth: &TelemetryAuth, +) { + fire( + PatchTelemetryEventType::CliUpdated, + "update", + serde_json::json!({ + "from_version": PACKAGE_VERSION, + "to_version": to_version, + "channel": channel, + "pinned": pinned, + "forced": forced, + }), + None::<&str>, + auth, + ) + .await; +} + +/// Track a failed self-update. `error_code` is the machine code the +/// `--json` envelope carries (e.g. `managed_install`, `check_failed`). +pub async fn track_cli_update_failed( + error_code: &str, + error: impl std::fmt::Display, + auth: &TelemetryAuth, +) { + fire( + PatchTelemetryEventType::CliUpdateFailed, + "update", + serde_json::json!({ "error_code": error_code }), + Some(error), + auth, + ) + .await; +} + #[cfg(test)] mod tests { use super::*; @@ -1013,8 +1154,30 @@ mod tests { let ecosystems = vec!["npm".to_string(), "pypi".to_string()]; let auth = TelemetryAuth::from_credentials(None, None); let mut pending = PendingTelemetry::new(); - track_patch_scanned(5, 3, 2, true, &ecosystems, true, &auth).await; - spawn_patch_scanned(&mut pending, 5, 3, 2, true, &ecosystems, true, &auth); + track_patch_scanned( + 5, + 3, + 2, + true, + &ecosystems, + true, + "hosted", + &["vendored"], + &auth, + ) + .await; + spawn_patch_scanned( + &mut pending, + 5, + 3, + 2, + true, + &ecosystems, + true, + "hosted", + &["vendored"], + &auth, + ); pending.flush().await; track_patch_scan_failed("all batches failed", true, &auth).await; spawn_patch_scan_failed(&mut pending, "all batches failed", true, &auth); @@ -1043,6 +1206,11 @@ mod tests { assert_eq!(bodies[0]["event_type"], "patch_scanned"); assert_eq!(bodies[0]["context"]["command"], "scan"); assert_eq!(bodies[0]["metadata"]["free_patches"], 3); + assert_eq!(bodies[0]["metadata"]["mode"], "hosted"); + assert_eq!( + bodies[0]["metadata"]["project_modes"], + serde_json::json!(["vendored"]) + ); assert_eq!(bodies[2]["event_type"], "patch_scan_failed"); assert_eq!(bodies[2]["context"]["command"], "scan"); assert_eq!(bodies[2]["error"]["message"], "all batches failed"); @@ -1218,6 +1386,217 @@ mod tests { "vex_generated" ); assert_eq!(PatchTelemetryEventType::VexFailed.as_str(), "vex_failed"); + // Self-update + assert_eq!(PatchTelemetryEventType::CliUpdated.as_str(), "cli_updated"); + assert_eq!( + PatchTelemetryEventType::CliUpdateFailed.as_str(), + "cli_update_failed" + ); + } + + /// The `event_type` strings depscan's `/v0/orgs/:slug/telemetry` + /// endpoint maps onto the `patch-lifecycle` ClickHouse category + /// (`PATCH_EVENT_TYPES_BY_STRING` in depscan's + /// `workspaces/api-v0/src/endpoints/orgs/telemetry-discriminators.ts`, + /// mirroring `TelemetryEventType` in + /// `workspaces/analytics-shared/src/telemetry/types.ts`). A type missing + /// there is still stored, but lands as `event_type = 'external'` and + /// drops out of every patch dashboard — so a new event type here needs + /// the depscan map extended first, then this list. + const DEPSCAN_PATCH_EVENT_TYPES: &[&str] = &[ + "patch_applied", + "patch_apply_failed", + "patch_removed", + "patch_remove_failed", + "autopatch_pr_created", + "autopatch_pr_failed", + "autopatch_pr_closed", + "autopatch_pr_merged", + "patch_searched", + "patch_downloaded", + "patch_rolled_back", + "patch_rollback_failed", + "patch_scanned", + "patch_scan_failed", + "patch_fetched", + "patch_fetch_failed", + "patch_vendored", + "patch_vendor_failed", + "patch_listed", + "patch_repaired", + "patch_repair_failed", + "patch_setup", + "vex_generated", + "vex_failed", + "cli_updated", + "cli_update_failed", + ]; + + #[test] + fn every_event_type_is_one_the_backend_maps() { + for event_type in PatchTelemetryEventType::ALL { + assert!( + DEPSCAN_PATCH_EVENT_TYPES.contains(&event_type.as_str()), + "{} is not in depscan's patch event map", + event_type.as_str() + ); + } + } + + /// The body shape depscan's v0 telemetry handler reads. It pulls + /// `event_sender_created_at` / `event_kind` / `artifact_purl` / + /// `input_purl` / `client_action` / `alert_action` / `user_agent` out + /// as columns and keeps the rest as extra-data JSON, deriving the + /// ClickHouse discriminators from `event_type` (and `context.source`). + /// Two of those matter here: + /// + /// * no `event_kind`: an event without one is "required" and is stored + /// even when the org has not opted into optional telemetry (anonymous + /// events go to the public patch org through the proxy). Sending + /// `event_kind: "informative"` would silently drop them. + /// * `event_sender_created_at` must parse with `Date.parse` and fall + /// within [now - 25h, now + 1h], or it is discarded for the receive + /// time. + #[test] + fn wire_shape_matches_the_backend_extractor() { + let mut metadata = HashMap::new(); + metadata.insert("patches_count".to_string(), serde_json::json!(2)); + let before = chrono_now_iso(); + let event = build_telemetry_event( + PatchTelemetryEventType::PatchApplyFailed, + "apply", + Some(metadata), + Some(("Error".to_string(), "boom".to_string())), + ); + let v = serde_json::to_value(&event).unwrap(); + let obj = v.as_object().unwrap(); + let mut keys: Vec<&str> = obj.keys().map(String::as_str).collect(); + keys.sort_unstable(); + assert_eq!( + keys, + [ + "context", + "error", + "event_sender_created_at", + "event_type", + "metadata", + "session_id" + ] + ); + for column in [ + "event_kind", + "artifact_purl", + "input_purl", + "client_action", + "alert_action", + "user_agent", + ] { + assert!( + !obj.contains_key(column), + "unexpected column field {column}" + ); + } + assert_eq!(v["event_type"], "patch_apply_failed"); + let mut ctx: Vec<&str> = v["context"] + .as_object() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + ctx.sort_unstable(); + assert_eq!(ctx, ["arch", "command", "platform", "version"]); + assert_eq!(v["error"]["type"], "Error"); + assert_eq!(v["error"]["message"], "boom"); + assert_eq!(v["metadata"]["patches_count"], 2); + assert!(uuid::Uuid::parse_str(v["session_id"].as_str().unwrap()).is_ok()); + + // Timestamp: fixed-width RFC 3339 UTC, so string order is time + // order, and taken at send time (well inside the window). + let after = chrono_now_iso(); + let ts = v["event_sender_created_at"].as_str().unwrap(); + assert!( + before.as_str() <= ts && ts <= after.as_str(), + "timestamp {ts} is not current ({before}..{after})" + ); + } + + /// Each public tracker posts the event type and metadata the backend + /// and its dashboards key on, to the org endpoint with the bearer. + #[tokio::test] + #[serial_test::serial] + async fn trackers_post_the_documented_events() { + use wiremock::matchers::{method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v0/orgs/acme/telemetry")) + .respond_with(ResponseTemplate::new(201)) + .mount(&server) + .await; + + let saved: Vec<(&str, Option)> = + ["SOCKET_TELEMETRY_DISABLED", "SOCKET_OFFLINE", "VITEST"] + .iter() + .map(|&k| (k, std::env::var(k).ok())) + .collect(); + for (key, _) in &saved { + std::env::remove_var(key); + } + + let auth = TelemetryAuth { + url: format!("{}/v0/orgs/acme/telemetry", server.uri()), + bearer: Some("tok".to_string()), + }; + track_patch_applied(3, false, &auth).await; + track_patch_hosted("scan", 2, 1, false, &auth).await; + track_patch_hosted_failed("get", "lockfile write failed", true, &auth).await; + track_cli_updated("9.9.9", "standalone", false, false, &auth).await; + track_cli_update_failed("managed_install", "managed by npm", &auth).await; + + for (key, value) in saved { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + + let reqs = server.received_requests().await.unwrap(); + let bodies: Vec = reqs + .iter() + .map(|r| { + assert_eq!(r.headers.get("authorization").unwrap(), "Bearer tok"); + serde_json::from_slice(&r.body).unwrap() + }) + .collect(); + assert_eq!(bodies.len(), 5); + + assert_eq!(bodies[0]["event_type"], "patch_applied"); + assert_eq!(bodies[0]["context"]["command"], "apply"); + assert_eq!(bodies[0]["metadata"]["mode"], "agent"); + assert_eq!(bodies[0]["metadata"]["patches_count"], 3); + + assert_eq!(bodies[1]["event_type"], "patch_applied"); + assert_eq!(bodies[1]["context"]["command"], "scan"); + assert_eq!(bodies[1]["metadata"]["mode"], "hosted"); + assert_eq!(bodies[1]["metadata"]["patches_count"], 2); + assert_eq!(bodies[1]["metadata"]["files_count"], 1); + + assert_eq!(bodies[2]["event_type"], "patch_apply_failed"); + assert_eq!(bodies[2]["context"]["command"], "get"); + assert_eq!(bodies[2]["metadata"]["mode"], "hosted"); + assert_eq!(bodies[2]["metadata"]["dry_run"], true); + assert_eq!(bodies[2]["error"]["message"], "lockfile write failed"); + + assert_eq!(bodies[3]["event_type"], "cli_updated"); + assert_eq!(bodies[3]["context"]["command"], "update"); + assert_eq!(bodies[3]["metadata"]["from_version"], PACKAGE_VERSION); + assert_eq!(bodies[3]["metadata"]["to_version"], "9.9.9"); + assert_eq!(bodies[3]["metadata"]["channel"], "standalone"); + + assert_eq!(bodies[4]["event_type"], "cli_update_failed"); + assert_eq!(bodies[4]["metadata"]["error_code"], "managed_install"); + assert_eq!(bodies[4]["error"]["message"], "managed by npm"); } #[test]