test(v10/e2e): Look up events via the organization trace endpoint

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/playwright.config.mjs

@@ -7,8 +7,8 @@ const expressPort = 3030;
  */
 const config = {
   testDir: './tests',
-  /* Maximum time one test can run for. */
-  timeout: 150_000,
+  /* Maximum time one test can run for. Spans take ~2min to become queryable via the trace endpoint. */
+  timeout: 210_000,
   expect: {
     /**
      * Maximum time expect() should wait for the condition to be met.

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/src/app.ts

@@ -1,14 +1,21 @@
 import * as Sentry from '@sentry/node';
 
 let lastTransactionId: string | undefined;
+let lastTransactionTraceId: string | undefined;
+let lastErrorTraceId: string | undefined;
 
 Sentry.init({
   environment: 'qa', // dynamic sampling bias to keep transactions
   dsn: process.env.E2E_TEST_DSN,
   includeLocalVariables: true,
   tracesSampleRate: 1,
+  beforeSend(event) {
+    lastErrorTraceId = event.contexts?.trace?.trace_id;
+    return event;
+  },
   beforeSendTransaction(event) {
     lastTransactionId = event.event_id;
+    lastTransactionTraceId = event.contexts?.trace?.trace_id;
     return event;
   },
 });
@@ -36,6 +43,7 @@ app.get('/test-transaction', function (req, res) {
 
     res.send({
       transactionId: lastTransactionId,
+      traceId: lastTransactionTraceId,
     });
   });
 });
@@ -45,7 +53,7 @@ app.get('/test-error', async function (req, res) {
 
   await Sentry.flush(2000);
 
-  res.send({ exceptionId });
+  res.send({ exceptionId, traceId: lastErrorTraceId });
 });
 
 app.get('/test-exception/:id', function (req, _res) {

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/tests/send-to-sentry.test.ts

@@ -1,45 +1,22 @@
 import { expect, test } from '@playwright/test';
-
-const EVENT_POLLING_TIMEOUT = 90_000;
-
-const authToken = process.env.E2E_TEST_AUTH_TOKEN;
-const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG;
-const sentryTestProject = process.env.E2E_TEST_SENTRY_PROJECT;
+import { EVENT_POLLING_OPTIONS, findErrorInTrace, findTransactionInTrace } from './utils/sentry-api';
 
 test('Sends exception to Sentry', async ({ baseURL }) => {
   const response = await fetch(`${baseURL}/test-error`);
-  const { exceptionId } = await response.json();
-
-  const url = `https://sentry.io/api/0/projects/${sentryTestOrgSlug}/${sentryTestProject}/events/${exceptionId}/`;
+  const { exceptionId, traceId } = await response.json();
 
-  console.log(`Polling for error eventId: ${exceptionId}`);
+  console.log(`Polling for error eventId: ${exceptionId} in trace: ${traceId}`);
 
-  await expect
-    .poll(
-      async () => {
-        const response = await fetch(url, { headers: { Authorization: `Bearer ${authToken}` } });
-        return response.status;
-      },
-      { timeout: EVENT_POLLING_TIMEOUT },
-    )
-    .toBe(200);
+  await expect.poll(() => findErrorInTrace(traceId, exceptionId), EVENT_POLLING_OPTIONS).toBeDefined();
 });
 
 test('Sends transaction to Sentry', async ({ baseURL }) => {
   const response = await fetch(`${baseURL}/test-transaction`);
-  const { transactionId } = await response.json();
-
-  const url = `https://sentry.io/api/0/projects/${sentryTestOrgSlug}/${sentryTestProject}/events/${transactionId}/`;
+  const { transactionId, traceId } = await response.json();
 
-  console.log(`Polling for transaction eventId: ${transactionId}`);
+  console.log(`Polling for transaction eventId: ${transactionId} in trace: ${traceId}`);
 
   await expect
-    .poll(
-      async () => {
-        const response = await fetch(url, { headers: { Authorization: `Bearer ${authToken}` } });
-        return response.status;
-      },
-      { timeout: EVENT_POLLING_TIMEOUT },
-    )
-    .toBe(200);
+    .poll(() => findTransactionInTrace(traceId, transactionId), EVENT_POLLING_OPTIONS)
+    .toMatchObject({ op: 'e2e-test' });
 });

dev-packages/e2e-tests/test-applications/node-express-send-to-sentry/tests/utils/sentry-api.ts

@@ -0,0 +1,72 @@
+const authToken = process.env.E2E_TEST_AUTH_TOKEN;
+const sentryTestOrgSlug = process.env.E2E_TEST_SENTRY_ORG_SLUG;
+
+/**
+ * Spans only become queryable once they have made it through to EAP, which takes
+ * noticeably longer than the error pipeline (~2min vs ~20s when this was measured).
+ */
+export const EVENT_POLLING_OPTIONS = { timeout: 180_000, intervals: [5_000] };
+
+/**
+ * A node of the span tree returned by the organization trace endpoint. Spans, errors and
+ * occurrences all share this shape and are discriminated by `event_type`.
+ */
+export interface TraceItem {
+  event_id?: string;
+  /** On spans this is the event id of the transaction the span belongs to. */
+  transaction_id?: string;
+  event_type?: 'span' | 'error' | 'occurrence' | 'uptime_check';
+  op?: string;
+  is_transaction?: boolean;
+  children?: TraceItem[];
+  errors?: TraceItem[];
+  occurrences?: TraceItem[];
+}
+
+export async function fetchTrace(traceId: string): Promise<TraceItem[]> {
+  const response = await fetch(
+    `https://sentry.io/api/0/organizations/${sentryTestOrgSlug}/trace/${traceId}/?statsPeriod=1h`,
+    { headers: { Authorization: `Bearer ${authToken}` } },
+  );
+
+  // The trace endpoint is org scoped, so the auth token needs `org:read` on top of the
+  // project scopes the other assertions rely on. That never resolves by waiting, so fail
+  // loudly instead of polling until the timeout and reporting it as a missing event.
+  if (response.status === 401 || response.status === 403) {
+    throw new Error(
+      `Trace lookup for ${traceId} was rejected with ${response.status}: ${await response.text()}. ` +
+        'E2E_TEST_AUTH_TOKEN needs the `org:read` scope.',
+    );
+  }
+
+  // Empty traces and the occasional rate limit are expected while polling, so treat anything
+  // else that is not a success as "not there yet" -- but log it, since a rejected request and
+  // a trace that has not landed are otherwise indistinguishable.
+  if (!response.ok) {
+    console.log(`Trace lookup for ${traceId} returned ${response.status}: ${await response.text()}`);
+    return [];
+  }
+
+  return await response.json();
+}
+
+/**
+ * Errors attach to whichever span was active when they were captured, and relocate from the
+ * top level into that span once it lands, so a given event can surface at any depth.
+ */
+export function flattenTrace(items: TraceItem[]): TraceItem[] {
+  return items.flatMap(item => [
+    item,
+    ...flattenTrace(item.children ?? []),
+    ...flattenTrace(item.errors ?? []),
+    ...flattenTrace(item.occurrences ?? []),
+  ]);
+}
+
+export async function findErrorInTrace(traceId: string, eventId: string): Promise<TraceItem | undefined> {
+  return flattenTrace(await fetchTrace(traceId)).find(item => item.event_type === 'error' && item.event_id === eventId);
+}
+
+export async function findTransactionInTrace(traceId: string, eventId: string): Promise<TraceItem | undefined> {
+  return flattenTrace(await fetchTrace(traceId)).find(item => item.is_transaction && item.transaction_id === eventId);
+}

dev-packages/e2e-tests/test-applications/react-send-to-sentry/playwright.config.mjs

@@ -5,8 +5,8 @@ import { devices } from '@playwright/test';
  */
 const config = {
   testDir: './tests',
-  /* Maximum time one test can run for. */
-  timeout: 150_000,
+  /* Maximum time one test can run for. Spans take ~2min to become queryable via the trace endpoint. */
+  timeout: 210_000,
   expect: {
     /**
      * Maximum time expect() should wait for the condition to be met.

dev-packages/e2e-tests/test-applications/react-send-to-sentry/src/globals.d.ts

@@ -1,5 +1,11 @@
+interface RecordedEvent {
+  eventId: string;
+  traceId: string;
+  op?: string;
+}
+
 interface Window {
-  recordedTransactions?: string[];
-  capturedExceptionId?: string;
+  recordedTransactions?: RecordedEvent[];
+  capturedException?: RecordedEvent;
   sentryReplayId?: string;
 }

dev-packages/e2e-tests/test-applications/react-send-to-sentry/src/index.tsx

@@ -44,16 +44,22 @@ Object.defineProperty(window, 'sentryReplayId', {
   },
 });
 
+// The trace id is recorded alongside the event id because events are looked up through the
+// organization trace endpoint, which is keyed by trace rather than by event.
 Sentry.addEventProcessor(event => {
-  if (
-    event.type === 'transaction' &&
-    (event.contexts?.trace?.op === 'pageload' || event.contexts?.trace?.op === 'navigation')
-  ) {
-    const eventId = event.event_id;
-    if (eventId) {
-      window.recordedTransactions = window.recordedTransactions || [];
-      window.recordedTransactions.push(eventId);
-    }
+  const eventId = event.event_id;
+  const traceId = event.contexts?.trace?.trace_id;
+  const op = event.contexts?.trace?.op;
+
+  if (!eventId || !traceId) {
+    return event;
+  }
+
+  if (event.type === 'transaction' && (op === 'pageload' || op === 'navigation')) {
+    window.recordedTransactions = window.recordedTransactions || [];
+    window.recordedTransactions.push({ eventId, traceId, op });
+  } else if (!event.type && event.exception) {
+    window.capturedException = { eventId, traceId };
   }
 
   return event;

dev-packages/e2e-tests/test-applications/react-send-to-sentry/src/pages/Index.tsx

@@ -10,8 +10,7 @@ const Index = () => {
         value="Capture Exception"
         id="exception-button"
         onClick={() => {
-          const eventId = Sentry.captureException(new Error('I am an error!'));
-          window.capturedExceptionId = eventId;
+          Sentry.captureException(new Error('I am an error!'));
         }}
       />
       <Link to="/user/5" id="navigation">