diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3884965e2..caef8e4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -446,7 +446,13 @@ jobs: echo "::warning::E2E tests skipped — STRIPE_API_KEY not available (fork PR?)" exit 0 fi - pnpm --filter @stripe/sync-e2e exec vitest run --exclude 'service-docker.test.ts' + pnpm --filter @stripe/sync-e2e exec vitest run \ + --exclude 'service-docker.test.ts' \ + --exclude 'test-e2e-network.test.ts' \ + --exclude 'test-server-all-api.test.ts' \ + --exclude 'test-server-sync.test.ts' \ + --exclude 'test-sync-e2e.test.ts' \ + --exclude 'test-sync-engine.test.ts' env: STRIPE_API_KEY: ${{ secrets.STRIPE_API_KEY }} POSTGRES_URL: 'postgres://postgres:postgres@localhost:55432/postgres' @@ -755,7 +761,7 @@ jobs: run: pnpm install --frozen-lockfile && pnpm build - name: CDN e2e tests - run: pnpm --filter @stripe/sync-e2e run test -- openapi-cdn.test.ts + run: pnpm --filter @stripe/sync-e2e exec vitest run openapi-cdn.test.ts env: STRIPE_SPEC_CDN_BASE_URL: ${{ needs.docs.outputs.deployment_url }}/stripe-api-specs diff --git a/apps/service/src/__tests__/workflow.test.ts b/apps/service/src/__tests__/workflow.test.ts index 3b67f449c..dd7b6f642 100644 --- a/apps/service/src/__tests__/workflow.test.ts +++ b/apps/service/src/__tests__/workflow.test.ts @@ -8,19 +8,25 @@ import { CONTINUE_AS_NEW_THRESHOLD } from '../lib/utils.js' type SourceInput = unknown -// workflowsPath points to the compiled workflow directory. -const workflowsPath = path.resolve(process.cwd(), 'dist/temporal/workflows') +// Point directly at the workflow index to avoid resolving the legacy dist/temporal/workflows.js file. +const workflowsPath = path.resolve(process.cwd(), 'dist/temporal/workflows/index.js') const emptyState = { streams: {}, global: {} } const noErrors: RunResult = { errors: [], state: emptyState } +const permanentSyncError: RunResult = { + errors: [ + { message: 'permanent sync failure', failure_type: 'system_error', stream: 'customers' }, + ], + state: emptyState, +} // Workflows now receive only the pipelineId string const testPipelineId = 'test_pipe' function stubActivities(overrides: Partial = {}): SyncActivities { - return { + const activities = { discoverCatalog: async () => ({ streams: [] }), - pipelineSetup: async () => ({}), + pipelineSetup: async () => {}, pipelineSync: async () => noErrors, readIntoQueue: async () => ({ count: 0, state: emptyState }), writeGoogleSheetsFromQueue: async () => ({ @@ -33,6 +39,13 @@ function stubActivities(overrides: Partial = {}): SyncActivities updatePipelineStatus: async () => {}, ...overrides, } + + return { + ...activities, + setup: activities.pipelineSetup, + sync: activities.pipelineSync, + teardown: activities.pipelineTeardown, + } as SyncActivities } /** Signal the workflow to delete. */ @@ -69,7 +82,6 @@ describe('pipelineWorkflow (unit — stubbed activities)', () => { activities: stubActivities({ pipelineSetup: async () => { setupCalled = true - return {} }, pipelineSync: async () => { runCallCount++ @@ -89,7 +101,7 @@ describe('pipelineWorkflow (unit — stubbed activities)', () => { await new Promise((r) => setTimeout(r, 2000)) const status = await handle.query('status') - expect(status.iteration).toBeGreaterThan(0) + expect((status as { iteration: number }).iteration).toBeGreaterThan(0) await signalDelete(handle) await handle.result() @@ -334,6 +346,82 @@ describe('pipelineWorkflow (unit — stubbed activities)', () => { }) }) + it('transitions to error instead of ready when reconcile returns permanent sync errors', async () => { + const statusWrites: string[] = [] + + const worker = await Worker.create({ + connection: testEnv.nativeConnection, + taskQueue: 'test-queue-3b-error', + workflowsPath, + activities: stubActivities({ + updatePipelineStatus: async (_id: string, status: string) => { + statusWrites.push(status) + }, + pipelineSync: async (_pipelineId: string, opts?) => { + if (opts?.input) return noErrors + return { ...permanentSyncError, eof: { reason: 'complete' as const } } + }, + }), + }) + + await worker.runUntil(async () => { + const handle = await testEnv.client.workflow.start('pipelineWorkflow', { + args: [testPipelineId], + workflowId: 'test-sync-3b-error', + taskQueue: 'test-queue-3b-error', + }) + + await new Promise((r) => setTimeout(r, 500)) + await signalDelete(handle) + await handle.result() + + expect(statusWrites).toContain('error') + expect(statusWrites).not.toContain('ready') + }) + }) + + it('retries transient sync activity failures and still reaches ready', async () => { + const statusWrites: string[] = [] + let reconcileCalls = 0 + + const worker = await Worker.create({ + connection: testEnv.nativeConnection, + taskQueue: 'test-queue-3b-retry', + workflowsPath, + activities: stubActivities({ + updatePipelineStatus: async (_id: string, status: string) => { + statusWrites.push(status) + }, + pipelineSync: async (_pipelineId: string, opts?) => { + if (opts?.input) return noErrors + + reconcileCalls++ + if (reconcileCalls === 1) { + throw new Error('transient sync failure') + } + + return { ...noErrors, eof: { reason: 'complete' as const } } + }, + }), + }) + + await worker.runUntil(async () => { + const handle = await testEnv.client.workflow.start('pipelineWorkflow', { + args: [testPipelineId], + workflowId: 'test-sync-3b-retry', + taskQueue: 'test-queue-3b-retry', + }) + + await new Promise((r) => setTimeout(r, 2500)) + await signalDelete(handle) + await handle.result() + + expect(reconcileCalls).toBeGreaterThanOrEqual(2) + expect(statusWrites).toContain('ready') + expect(statusWrites).not.toContain('error') + }) + }) + it('queues live events while paused and drains them after resume', async () => { const syncCalls: { input?: SourceInput[] }[] = [] @@ -464,7 +552,6 @@ describe('pipelineWorkflow (unit — stubbed activities)', () => { activities: stubActivities({ pipelineSetup: async () => { setupCalls++ - return {} }, pipelineSync: async () => { syncCallCount++ diff --git a/apps/service/src/temporal/activities/_shared.ts b/apps/service/src/temporal/activities/_shared.ts index 022a2ef9a..ff1c3c506 100644 --- a/apps/service/src/temporal/activities/_shared.ts +++ b/apps/service/src/temporal/activities/_shared.ts @@ -4,6 +4,7 @@ import { createRemoteEngine } from '@stripe/sync-engine' import { Kafka } from 'kafkajs' import type { Producer } from 'kafkajs' import type { PipelineStore } from '../../lib/stores.js' +import type { SyncRunError } from '../sync-errors.js' export interface ActivitiesContext { /** Remote engine client — satisfies the {@link Engine} interface over HTTP. Drop-in replacement for a local engine. */ @@ -104,7 +105,7 @@ export function createActivitiesContext(opts: { } export interface RunResult { - errors: Array<{ message: string; failure_type?: string; stream?: string }> + errors: SyncRunError[] state: SourceState } diff --git a/apps/service/src/temporal/activities/pipeline-sync.ts b/apps/service/src/temporal/activities/pipeline-sync.ts index 030a7d9fd..7d2596a88 100644 --- a/apps/service/src/temporal/activities/pipeline-sync.ts +++ b/apps/service/src/temporal/activities/pipeline-sync.ts @@ -1,6 +1,8 @@ +import { ApplicationFailure } from '@temporalio/activity' import type { SourceInputMessage, SourceReadOptions } from '@stripe/sync-engine' import type { ActivitiesContext } from './_shared.js' import { asIterable, drainMessages, type RunResult } from './_shared.js' +import { classifySyncErrors, summarizeSyncErrors } from '../sync-errors.js' export function createPipelineSyncActivity(context: ActivitiesContext) { return async function pipelineSync( @@ -28,6 +30,13 @@ export function createPipelineSyncActivity(context: ActivitiesContext) { destination: { type, [type]: destConfig }, }) } + const { transient, permanent } = classifySyncErrors(errors) + if (permanent.length > 0) { + return { errors, state, eof } + } + if (transient.length > 0) { + throw ApplicationFailure.retryable(summarizeSyncErrors(transient), 'TransientSyncError') + } return { errors, state, eof } } } diff --git a/apps/service/src/temporal/sync-errors.ts b/apps/service/src/temporal/sync-errors.ts new file mode 100644 index 000000000..b20cedf98 --- /dev/null +++ b/apps/service/src/temporal/sync-errors.ts @@ -0,0 +1,37 @@ +export type SyncRunError = { + message: string + failure_type?: string + stream?: string +} + +export type ClassifiedSyncErrors = { + transient: SyncRunError[] + permanent: SyncRunError[] +} + +const PERMANENT_FAILURE_TYPES = new Set(['system_error', 'config_error']) + +export function classifySyncErrors(errors: SyncRunError[]): ClassifiedSyncErrors { + const transient: SyncRunError[] = [] + const permanent: SyncRunError[] = [] + + for (const error of errors) { + if (PERMANENT_FAILURE_TYPES.has(error.failure_type ?? '')) { + permanent.push(error) + } else { + transient.push(error) + } + } + + return { transient, permanent } +} + +export function summarizeSyncErrors(errors: SyncRunError[]): string { + return errors + .map((error) => { + const failureType = error.failure_type ?? 'unknown_error' + const stream = error.stream ? `/${error.stream}` : '' + return `[${failureType}${stream}] ${error.message}` + }) + .join('; ') +} diff --git a/apps/service/src/temporal/workflows/pipeline-workflow.ts b/apps/service/src/temporal/workflows/pipeline-workflow.ts index 73d5dfc21..299531cda 100644 --- a/apps/service/src/temporal/workflows/pipeline-workflow.ts +++ b/apps/service/src/temporal/workflows/pipeline-workflow.ts @@ -3,6 +3,7 @@ import { condition, continueAsNew, setHandler } from '@temporalio/workflow' import type { SourceInputMessage, SourceState } from '@stripe/sync-protocol' import type { DesiredStatus, PipelineStatus } from '../../lib/createSchemas.js' import { CONTINUE_AS_NEW_THRESHOLD } from '../../lib/utils.js' +import { classifySyncErrors } from '../sync-errors.js' import { desiredStatusSignal, pipelineSetup, @@ -22,6 +23,7 @@ export type TeardownState = 'started' | 'completed' export interface PipelineWorkflowState { phase?: ReconcileState paused?: boolean + errored?: boolean setup?: SetupState teardown?: TeardownState } @@ -42,6 +44,7 @@ export async function pipelineWorkflow( let desiredStatus: DesiredStatus = opts?.desiredStatus ?? 'active' let sourceState: SourceState = opts?.sourceState ?? { streams: {}, global: {} } let state: PipelineWorkflowState = { ...opts?.state } + let desiredStatusSignalCount = 0 // Transient workflow-local state. let operationCount = 0 @@ -51,12 +54,14 @@ export async function pipelineWorkflow( }) setHandler(desiredStatusSignal, (status: DesiredStatus) => { desiredStatus = status + desiredStatusSignalCount++ }) // MARK: - State function derivePipelineStatus(): PipelineStatus { if (state.teardown) return 'teardown' + if (state.errored) return 'error' if (state.paused) return 'paused' if (state.setup !== 'completed') return 'setup' return state.phase === 'ready' ? 'ready' : 'backfill' @@ -77,7 +82,21 @@ export async function pipelineWorkflow( * no longer active or because the workflow should roll over into continue-as-new. */ function runInterrupted() { - return desiredStatus !== 'active' || operationCount >= CONTINUE_AS_NEW_THRESHOLD + return ( + desiredStatus !== 'active' || operationCount >= CONTINUE_AS_NEW_THRESHOLD || !!state.errored + ) + } + + async function markPermanentError(): Promise { + await setState({ errored: true, phase: 'backfilling' }) + } + + async function waitForErrorRecovery(): Promise { + const signalCount = desiredStatusSignalCount + await condition(() => desiredStatus === 'deleted' || desiredStatusSignalCount > signalCount) + if (desiredStatus === 'active') { + await setState({ errored: false, phase: 'backfilling' }) + } } // MARK: - Live loop @@ -97,8 +116,12 @@ export async function pipelineWorkflow( const events = await waitForLiveEvents() if (!events) return - await pipelineSync(pipelineId, { input: events }) + const result = await pipelineSync(pipelineId, { input: events }) operationCount++ + if (classifySyncErrors(result.errors).permanent.length > 0) { + await markPermanentError() + return + } } } @@ -127,11 +150,15 @@ export async function pipelineWorkflow( state_limit: 100, time_limit: 10, }) + operationCount++ sourceState = result.state - if (result.eof?.reason === 'complete') { + if (classifySyncErrors(result.errors).permanent.length > 0) { + await markPermanentError() + return + } + if (result.eof?.reason === 'complete' && !state.errored) { await setState({ phase: 'ready' }) } - operationCount++ } } @@ -144,6 +171,11 @@ export async function pipelineWorkflow( } while (desiredStatus !== 'deleted') { + if (state.errored) { + await waitForErrorRecovery() + continue + } + if (desiredStatus === 'paused') { await setState({ paused: true }) await condition(() => desiredStatus !== 'paused') diff --git a/compose.dev.yml b/compose.dev.yml index 316881181..046ddee70 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -6,6 +6,7 @@ services: engine: + image: sync-engine-engine:dev build: context: . target: engine @@ -21,6 +22,7 @@ services: start_period: 10s service: + image: sync-engine-service:dev build: context: . target: service @@ -47,9 +49,7 @@ services: start_period: 10s worker: - build: - context: . - target: service + image: sync-engine-service:dev command: - worker - --temporal-address diff --git a/e2e/compose.e2e.yml b/e2e/compose.e2e.yml new file mode 100644 index 000000000..a87c0df95 --- /dev/null +++ b/e2e/compose.e2e.yml @@ -0,0 +1,46 @@ +# E2E overlay — layers on top of compose.yml + compose.dev.yml. +# Adds --data-dir (required by CLI), shared pipeline volume, and +# extra_hosts so containers can reach host-bound test servers. +# +# Usage: +# docker compose -f compose.yml -f compose.dev.yml -f e2e/compose.e2e.yml up --build -d + +volumes: + pipeline-data: + +services: + engine: + extra_hosts: + - 'host.docker.internal:host-gateway' + + service: + extra_hosts: + - 'host.docker.internal:host-gateway' + volumes: + - pipeline-data:/data + command: + - serve + - --temporal-address + - temporal:7233 + - --temporal-task-queue + - sync-engine + - --data-dir + - /data/pipelines + - --port + - '4020' + + worker: + extra_hosts: + - 'host.docker.internal:host-gateway' + volumes: + - pipeline-data:/data + command: + - worker + - --temporal-address + - temporal:7233 + - --temporal-task-queue + - sync-engine + - --engine-url + - http://engine:3000 + - --data-dir + - /data/pipelines diff --git a/e2e/package.json b/e2e/package.json index 518d1ab38..eccec4a5b 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -14,7 +14,9 @@ "@stripe/sync-protocol": "workspace:*", "@stripe/sync-service": "workspace:*", "@stripe/sync-source-stripe": "workspace:*", + "@stripe/sync-test-utils": "workspace:*", "@temporalio/client": "^1", + "@temporalio/testing": "^1.15.0", "@temporalio/worker": "^1", "@types/pg": "^8.20.0", "googleapis": "^144", diff --git a/e2e/service-docker.test.ts b/e2e/service-docker.test.ts index f911b321e..48fa6c9e4 100644 --- a/e2e/service-docker.test.ts +++ b/e2e/service-docker.test.ts @@ -23,6 +23,7 @@ const REPO_ROOT = path.resolve(import.meta.dirname, '..') const COMPOSE_CMD = `docker compose -f compose.yml -f compose.dev.yml` const SKIP_CLEANUP = process.env.SKIP_CLEANUP === '1' +const STOP_MANAGED_STACK = process.env.STOP_MANAGED_STACK === '1' // When true, skip building and starting containers (CI pre-starts them). const SKIP_SETUP = process.env.SKIP_SETUP === '1' @@ -71,12 +72,14 @@ describeWithEnv( beforeAll(async () => { schema = `docker_e2e_${Date.now()}` - if (SKIP_SETUP || (await isServiceHealthy())) { - console.log('\n Service already healthy — skipping build & container startup') + if (SKIP_SETUP) { + console.log('\n SKIP_SETUP=1 — assuming containers are already managed externally') } else { + if (await isServiceHealthy()) { + console.log('\n Service already healthy — reconciling shared stack') + } managedContainers = true - // 1. Build TypeScript so Dockerfiles have fresh dist/ console.log('\n Building packages...') execSync('pnpm build', { cwd: REPO_ROOT, stdio: 'pipe' }) @@ -100,6 +103,7 @@ describeWithEnv( console.log(` Schema: ${schema}`) console.log(` Postgres: ${POSTGRES_HOST_URL}`) console.log(` Cleanup: ${SKIP_CLEANUP ? 'no (SKIP_CLEANUP=1)' : 'yes'}`) + console.log(` Stop stack: ${STOP_MANAGED_STACK ? 'yes' : 'no'}`) }, 5 * 60_000) // 5 min — includes docker build afterAll(async () => { @@ -109,7 +113,7 @@ describeWithEnv( await pool?.end().catch(() => {}) // Only stop containers we started - if (managedContainers) { + if (managedContainers && STOP_MANAGED_STACK) { execSync(`${COMPOSE_CMD} stop engine service worker`, { cwd: REPO_ROOT, stdio: 'pipe' }) execSync(`${COMPOSE_CMD} rm -f engine service worker`, { cwd: REPO_ROOT, stdio: 'pipe' }) } diff --git a/e2e/test-e2e-network.test.ts b/e2e/test-e2e-network.test.ts new file mode 100644 index 000000000..227dda345 --- /dev/null +++ b/e2e/test-e2e-network.test.ts @@ -0,0 +1,338 @@ +import { describe, expect, it } from 'vitest' +import createFetchClient from 'openapi-fetch' +import type { paths } from '../apps/service/src/__generated__/openapi.js' +import { BUNDLED_API_VERSION } from '../packages/openapi/src/versions.js' +import { + SERVICE_URL, + pauseComposeService, + pauseDockerContainer, + pollUntil, + startServiceHarness, + unpauseComposeService, + unpauseDockerContainer, + type ServiceHarness, +} from './test-server-harness.js' + +type PipelineRecord = { id: string; status: string; desired_status: string } + +const api = () => createFetchClient({ baseUrl: SERVICE_URL }) + +let schemaCounter = 0 + +function uniqueSchema(prefix: string): string { + return `${prefix}_${Date.now()}_${schemaCounter++}` +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +async function countRows(harness: ServiceHarness, schema: string): Promise { + try { + const { rows } = await harness.destPool.query<{ n: number }>( + `SELECT count(*)::int AS n FROM "${schema}"."customers"` + ) + return rows[0]?.n ?? 0 + } catch (err) { + if ((err as { code?: string })?.code === '42P01') return 0 + throw err + } +} + +async function getPipeline(id: string): Promise { + const { data } = await api().GET('/pipelines/{id}', { + params: { path: { id } }, + }) + return (data as PipelineRecord | undefined) ?? null +} + +async function createCustomersPipeline( + harness: ServiceHarness, + schema: string, + sourceOverrides: Record = {} +): Promise { + const { data, error } = await api().POST('/pipelines', { + body: { + source: { + type: 'stripe', + stripe: { + api_key: 'sk_test_fake', + api_version: BUNDLED_API_VERSION, + base_url: harness.testServerContainerUrl(), + rate_limit: 1000, + ...sourceOverrides, + }, + }, + destination: { + type: 'postgres', + postgres: { + connection_string: harness.destPgContainerUrl(), + schema, + }, + }, + streams: [{ name: 'customers' }], + } as never, + }) + expect(error).toBeUndefined() + expect(data?.id).toMatch(/^pipe_/) + return data!.id +} + +async function deletePipeline(id: string | undefined): Promise { + if (!id) return + await api() + .DELETE('/pipelines/{id}', { + params: { path: { id } }, + }) + .catch(() => undefined) +} + +async function cleanupHarness( + harness: ServiceHarness | undefined, + pipelineId: string | undefined, + schema: string +): Promise { + await deletePipeline(pipelineId) + await harness?.destPool?.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => {}) + await harness?.close().catch(() => {}) +} + +async function waitForPartialRows( + harness: ServiceHarness, + schema: string, + expectedCount: number, + minimumRows = 100 +): Promise { + await pollUntil( + async () => { + const count = await countRows(harness, schema) + return count >= minimumRows && count < expectedCount + }, + { timeout: 60_000, interval: 1000 } + ) + return countRows(harness, schema) +} + +async function waitForCompletionWithoutFalseReady(opts: { + harness: ServiceHarness + pipelineId: string + schema: string + expectedCount: number + timeout?: number +}): Promise { + const deadline = Date.now() + (opts.timeout ?? 90_000) + while (Date.now() < deadline) { + const [pipeline, rows] = await Promise.all([ + getPipeline(opts.pipelineId), + countRows(opts.harness, opts.schema), + ]) + + if (pipeline?.status === 'ready' && rows < opts.expectedCount) { + throw new Error( + `pipeline ${opts.pipelineId} reached ready with only ${rows}/${opts.expectedCount} rows` + ) + } + if (rows === opts.expectedCount) { + return + } + + await sleep(1000) + } + + const finalRows = await countRows(opts.harness, opts.schema) + const finalPipeline = await getPipeline(opts.pipelineId) + throw new Error( + `pipeline ${opts.pipelineId} did not finish: status=${finalPipeline?.status ?? 'missing'} rows=${finalRows}/${opts.expectedCount}` + ) +} + +async function waitForStalledIncompletePipeline(opts: { + harness: ServiceHarness + pipelineId: string + schema: string + expectedCount: number + stableForMs?: number + timeout?: number +}): Promise<{ rows: number; status: string | null }> { + const stableForMs = opts.stableForMs ?? 8000 + const deadline = Date.now() + (opts.timeout ?? 45_000) + let lastRows = -1 + let lastChangeAt = Date.now() + + while (Date.now() < deadline) { + const [pipeline, rows] = await Promise.all([ + getPipeline(opts.pipelineId), + countRows(opts.harness, opts.schema), + ]) + + if (pipeline?.status === 'ready' && rows < opts.expectedCount) { + throw new Error( + `pipeline ${opts.pipelineId} reached ready with only ${rows}/${opts.expectedCount} rows after interruption` + ) + } + if (rows === opts.expectedCount) { + throw new Error(`pipeline ${opts.pipelineId} unexpectedly completed after source shutdown`) + } + + if (rows !== lastRows) { + lastRows = rows + lastChangeAt = Date.now() + } else if (Date.now() - lastChangeAt >= stableForMs) { + return { rows, status: pipeline?.status ?? null } + } + + await sleep(1000) + } + + const finalRows = await countRows(opts.harness, opts.schema) + const finalPipeline = await getPipeline(opts.pipelineId) + throw new Error( + `pipeline ${opts.pipelineId} never settled after interruption: status=${finalPipeline?.status ?? 'missing'} rows=${finalRows}/${opts.expectedCount}` + ) +} + +describe('network interruption e2e via Docker service', () => { + it('recovers from a transient list-server 500 without reporting ready early', async () => { + const schema = uniqueSchema('e2e_network_http_500') + let harness: ServiceHarness | undefined + let pipelineId: string | undefined + + try { + harness = await startServiceHarness({ + customerCount: 250, + listServer: { + failures: [ + { + path: '/v1/customers', + status: 500, + after: 1, + times: 1, + stripeError: { + type: 'api_error', + message: 'Injected transient customers page failure', + }, + }, + ], + }, + }) + + pipelineId = await createCustomersPipeline(harness, schema, { + backfill_concurrency: 1, + rate_limit: 1000, + }) + + await waitForCompletionWithoutFalseReady({ + harness, + pipelineId, + schema, + expectedCount: harness.expectedIds.length, + timeout: 120_000, + }) + + const pipeline = await getPipeline(pipelineId) + expect(pipeline?.status).toBe('ready') + expect(await countRows(harness, schema)).toBe(harness.expectedIds.length) + } finally { + await cleanupHarness(harness, pipelineId, schema) + } + }, 180_000) + + it('does not report ready after the source server disappears mid-backfill', async () => { + const schema = uniqueSchema('e2e_network_source_down') + let harness: ServiceHarness | undefined + let pipelineId: string | undefined + + try { + harness = await startServiceHarness({ customerCount: 400 }) + pipelineId = await createCustomersPipeline(harness, schema, { + backfill_concurrency: 1, + rate_limit: 1, + }) + + await waitForPartialRows(harness, schema, harness.expectedIds.length) + await harness.testServer.close() + + const stalled = await waitForStalledIncompletePipeline({ + harness, + pipelineId, + schema, + expectedCount: harness.expectedIds.length, + }) + + expect(stalled.rows).toBeLessThan(harness.expectedIds.length) + expect(stalled.status).not.toBe('ready') + } finally { + await cleanupHarness(harness, pipelineId, schema) + } + }, 180_000) + + it('resumes after destination Postgres is paused mid-sync', async () => { + const schema = uniqueSchema('e2e_network_dest_pg_pause') + let harness: ServiceHarness | undefined + let pipelineId: string | undefined + + try { + harness = await startServiceHarness({ customerCount: 400 }) + pipelineId = await createCustomersPipeline(harness, schema, { + backfill_concurrency: 1, + rate_limit: 1, + }) + + await waitForPartialRows(harness, schema, harness.expectedIds.length) + + pauseDockerContainer(harness.destDocker.containerId) + try { + await sleep(4000) + const pipeline = await getPipeline(pipelineId) + expect(pipeline?.status).not.toBe('ready') + } finally { + unpauseDockerContainer(harness.destDocker.containerId) + } + + await waitForCompletionWithoutFalseReady({ + harness, + pipelineId, + schema, + expectedCount: harness.expectedIds.length, + timeout: 120_000, + }) + } finally { + await cleanupHarness(harness, pipelineId, schema) + } + }, 180_000) + + it('resumes after the engine container is paused mid-sync', async () => { + const schema = uniqueSchema('e2e_network_engine_pause') + let harness: ServiceHarness | undefined + let pipelineId: string | undefined + + try { + harness = await startServiceHarness({ customerCount: 400 }) + pipelineId = await createCustomersPipeline(harness, schema, { + backfill_concurrency: 1, + rate_limit: 1, + }) + + const rowsBeforePause = await waitForPartialRows(harness, schema, harness.expectedIds.length) + + pauseComposeService('engine') + try { + await sleep(4000) + const rowsDuringPause = await countRows(harness, schema) + expect(rowsDuringPause).toBe(rowsBeforePause) + } finally { + unpauseComposeService('engine') + } + + await waitForCompletionWithoutFalseReady({ + harness, + pipelineId, + schema, + expectedCount: harness.expectedIds.length, + timeout: 120_000, + }) + } finally { + await cleanupHarness(harness, pipelineId, schema) + } + }, 180_000) +}) diff --git a/e2e/test-server-all-api.test.ts b/e2e/test-server-all-api.test.ts new file mode 100644 index 000000000..687ffc6f1 --- /dev/null +++ b/e2e/test-server-all-api.test.ts @@ -0,0 +1,402 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { execSync } from 'node:child_process' +import pg from 'pg' +import { + applyCreatedTimestampRange, + createStripeListServer, + ensureObjectTable, + quoteIdentifier, + resolveEndpointSet, + startDockerPostgres18, + type DockerPostgres18Handle, + type StripeListServer, +} from '@stripe/sync-test-utils' +import { createConnectorResolver, createEngine, type PipelineConfig } from '@stripe/sync-engine' +import { + SUPPORTED_API_VERSIONS, + resolveOpenApiSpec, + findSchemaNameByResourceId, + generateObjectsFromSchema, +} from '@stripe/sync-openapi' +import destinationPostgres from '@stripe/sync-destination-postgres' +import sourceStripe, { type StripeStreamState } from '@stripe/sync-source-stripe' +import { utc } from './test-server-harness.js' + +const SOURCE_SCHEMA = 'stripe' +const OBJECTS_PER_STREAM = 1200 +const RATE_LIMIT = 100000 + +const RANGE_START = utc('2025-01-01') +const RANGE_END = utc('2026-01-01') + +let sourceDocker: DockerPostgres18Handle +let destDocker: DockerPostgres18Handle +let testServer: StripeListServer +let sourcePool: pg.Pool +let destPool: pg.Pool +const specPathByVersion = new Map() +let githubToken: string | null | undefined + +type StreamSeed = { + tableName: string + objectIds: string[] +} + +const INSERT_BATCH = 1000 + +async function replaceTableObjects( + tableName: string, + objects: Record[] +): Promise { + await ensureObjectTable(sourcePool, SOURCE_SCHEMA, tableName) + const q = quoteIdentifier + const table = `${q(SOURCE_SCHEMA)}.${q(tableName)}` + + const client = await sourcePool.connect() + try { + await client.query('BEGIN') + await client.query(`TRUNCATE TABLE ${table}`) + for (let i = 0; i < objects.length; i += INSERT_BATCH) { + const batch = objects.slice(i, i + INSERT_BATCH) + const values: string[] = [] + const placeholders: string[] = [] + for (const obj of batch) { + values.push(JSON.stringify(obj)) + placeholders.push(`($${values.length}::jsonb)`) + } + await client.query( + `INSERT INTO ${table} ("_raw_data") VALUES ${placeholders.join(',')}`, + values + ) + } + await client.query('COMMIT') + } catch (err) { + await client.query('ROLLBACK') + throw err + } finally { + client.release() + } +} + +function schemaForVersion(apiVersion: string): string { + const safeVersion = apiVersion.toLowerCase().replace(/[^a-z0-9]+/g, '_') + return `all_api_${safeVersion}_${Date.now()}` +} + +function getGithubToken(): string | null { + if (githubToken !== undefined) return githubToken + + try { + const token = execSync('gh auth token', { + cwd: new URL('..', import.meta.url).pathname, + stdio: 'pipe', + encoding: 'utf8', + }).trim() + githubToken = token.length > 0 ? token : null + } catch { + githubToken = null + } + + return githubToken +} + +async function specFetch(input: RequestInfo | URL, init?: RequestInit): Promise { + const url = + typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url + const token = getGithubToken() + + if (!token || !url.startsWith('https://api.github.com/')) { + return fetch(input, init) + } + + const headers = new Headers(init?.headers) + headers.set('Authorization', `Bearer ${token}`) + headers.set('X-GitHub-Api-Version', '2022-11-28') + + return fetch(input, { ...init, headers }) +} + +async function resolveSpecPath(apiVersion: string): Promise { + const cached = specPathByVersion.get(apiVersion) + if (cached) return cached + + const resolved = await resolveOpenApiSpec({ apiVersion }, specFetch) + if (!resolved.cachePath) { + throw new Error(`No cache path returned for Stripe API version ${apiVersion}`) + } + + specPathByVersion.set(apiVersion, resolved.cachePath) + return resolved.cachePath +} + +async function syncAllEndpointsForVersion(apiVersion: string): Promise { + const createdRange = { startUnix: RANGE_START, endUnix: RANGE_END } + const openApiSpecPath = await resolveSpecPath(apiVersion) + const endpointSet = await resolveEndpointSet({ + apiVersion, + openApiSpecPath, + fetchImpl: specFetch, + }) + const sortedEndpoints = [...endpointSet.endpoints.values()].sort((a, b) => + a.tableName.localeCompare(b.tableName) + ) + + const seededStreams: StreamSeed[] = [] + const destSchema = schemaForVersion(apiVersion) + const versionTestServer = await createStripeListServer({ + postgresUrl: sourceDocker.connectionString, + host: '127.0.0.1', + port: 0, + accountCreated: RANGE_START, + logRequests: false, + validateQueryParams: true, + apiVersion, + openApiSpecPath, + fetchImpl: specFetch, + }) + + expect(sortedEndpoints.length, `${apiVersion} should expose at least one stream`).toBeGreaterThan( + 0 + ) + + try { + const seedable = sortedEndpoints.filter( + (ep) => findSchemaNameByResourceId(endpointSet.spec, ep.resourceId) != null + ) + + const SEED_CONCURRENCY = 8 + for (let i = 0; i < seedable.length; i += SEED_CONCURRENCY) { + const batch = seedable.slice(i, i + SEED_CONCURRENCY) + await Promise.all( + batch.map(async (endpoint) => { + const schemaName = findSchemaNameByResourceId(endpointSet.spec, endpoint.resourceId)! + const objects = applyCreatedTimestampRange( + generateObjectsFromSchema(endpointSet.spec, schemaName, OBJECTS_PER_STREAM, { + tableName: endpoint.tableName, + }), + createdRange + ) + + await replaceTableObjects(endpoint.tableName, objects) + + seededStreams.push({ + tableName: endpoint.tableName, + objectIds: objects.map((object: Record) => object.id as string), + }) + }) + ) + } + + const pipeline: PipelineConfig = { + source: { + type: 'stripe', + stripe: { + api_key: 'sk_test_fake', + api_version: endpointSet.apiVersion, + base_url: versionTestServer.url, + rate_limit: RATE_LIMIT, + backfill_concurrency: 12, + }, + }, + destination: { + type: 'postgres', + postgres: { + connection_string: destDocker.connectionString, + schema: destSchema, + batch_size: 100, + }, + }, + streams: seededStreams.map((stream) => ({ + name: stream.tableName, + sync_mode: 'full_refresh', + })), + } + + const resolver = await createConnectorResolver({ + sources: { stripe: sourceStripe }, + destinations: { postgres: destinationPostgres }, + }) + const engine = await createEngine(resolver) + + const finalState: Record = {} + + for await (const setupMsg of engine.pipeline_setup(pipeline)) { + void setupMsg + } + + const syncStart = performance.now() + console.error( + ` [${apiVersion}] pipeline: running pipeline_sync (${seededStreams.length} streams)` + ) + for await (const msg of engine.pipeline_sync(pipeline)) { + if (msg.type === 'source_state' && msg.source_state.state_type === 'stream') { + finalState[msg.source_state.stream] = msg.source_state.data + } + } + console.error(` [${apiVersion}] pipeline: sync done in ${ms(syncStart)}`) + + const failures: string[] = [] + const syncedCounts: string[] = [] + + for (const seed of seededStreams) { + const { rows } = await destPool.query<{ id: string }>( + `SELECT id FROM ${quoteIdentifier(destSchema)}.${quoteIdentifier(seed.tableName)} ORDER BY id` + ) + + syncedCounts.push( + ` ${seed.tableName}: ${rows.length} synced (${seed.objectIds.length} seeded)` + ) + + const destIds = new Set(rows.map((row) => row.id)) + const expectedIds = new Set(seed.objectIds) + const missing = [...expectedIds].filter((id) => !destIds.has(id)) + const unexpected = [...destIds].filter((id) => !expectedIds.has(id)) + + if (missing.length > 0) { + failures.push( + `${apiVersion}/${seed.tableName}: missing ${missing.length} objects (first 5: ${missing.slice(0, 5).join(', ')})` + ) + } + if (unexpected.length > 0) { + failures.push( + `${apiVersion}/${seed.tableName}: unexpected ${unexpected.length} objects (first 5: ${unexpected.slice(0, 5).join(', ')})` + ) + } + if (rows.length !== seed.objectIds.length) { + failures.push( + `${apiVersion}/${seed.tableName}: expected ${seed.objectIds.length} rows, got ${rows.length}` + ) + } + + const streamState = finalState[seed.tableName] as StripeStreamState | undefined + if (streamState?.status !== 'complete') { + failures.push( + `${apiVersion}/${seed.tableName}: final state was ${streamState?.status ?? 'missing'}` + ) + } + } + + // console.log( + // `\n [${apiVersion}] ${seededStreams.length} streams:\n${syncedCounts.join('\n')}\n` + // ) + + expect(failures, failures.join('\n')).toHaveLength(0) + } finally { + await destPool.query(`DROP SCHEMA IF EXISTS ${quoteIdentifier(destSchema)} CASCADE`) + await versionTestServer.close().catch(() => {}) + } +} + +function ms(since: number): string { + const elapsed = performance.now() - since + return elapsed < 1000 ? `${elapsed.toFixed(0)}ms` : `${(elapsed / 1000).toFixed(1)}s` +} + +describe('test-server API', () => { + beforeAll(async () => { + const [src, dest] = await Promise.all([startDockerPostgres18(), startDockerPostgres18()]) + + sourceDocker = src + destDocker = dest + + sourcePool = new pg.Pool({ connectionString: sourceDocker.connectionString }) + destPool = new pg.Pool({ connectionString: destDocker.connectionString }) + sourcePool.on('error', () => {}) + destPool.on('error', () => {}) + + testServer = await createStripeListServer({ + postgresUrl: sourceDocker.connectionString, + host: '127.0.0.1', + port: 0, + accountCreated: RANGE_START, + logRequests: false, + validateQueryParams: true, + }) + }, 10 * 60_000) + + afterAll(async () => { + await testServer?.close().catch(() => {}) + await sourcePool?.end().catch(() => {}) + await destPool?.end().catch(() => {}) + await destDocker?.stop() + await sourceDocker?.stop() + }, 60_000) + + it('retrieve returns object by ID, 404 for missing', async () => { + await replaceTableObjects('customers', [ + { id: 'cus_ret_1', object: 'customer', created: RANGE_START + 100 }, + ]) + + const okRes = await fetch(`${testServer.url}/v1/customers/cus_ret_1`, { + headers: { Authorization: 'Bearer sk_test_fake' }, + }) + expect(okRes.status).toBe(200) + const body = (await okRes.json()) as Record + expect(body.id).toBe('cus_ret_1') + expect(body.object).toBe('customer') + + const missingRes = await fetch(`${testServer.url}/v1/customers/cus_nonexistent`, { + headers: { Authorization: 'Bearer sk_test_fake' }, + }) + expect(missingRes.status).toBe(404) + const errBody = (await missingRes.json()) as { error: { code: string } } + expect(errBody.error.code).toBe('resource_missing') + }, 120_000) + + it('unrecognized path returns 404, non-GET returns 405', async () => { + await replaceTableObjects('customers', []) + + const notFoundRes = await fetch(`${testServer.url}/v1/totally_fake_endpoint`, { + headers: { Authorization: 'Bearer sk_test_fake' }, + }) + expect(notFoundRes.status).toBe(404) + const errBody = (await notFoundRes.json()) as { error: { type: string } } + expect(errBody.error.type).toBe('invalid_request_error') + + const methodRes = await fetch(`${testServer.url}/v1/customers`, { + method: 'POST', + headers: { Authorization: 'Bearer sk_test_fake' }, + }) + expect(methodRes.status).toBe(405) + }, 120_000) + + it('list request with invalid query params fails', async () => { + const validatingServer = await createStripeListServer({ + postgresUrl: sourceDocker.connectionString, + host: '127.0.0.1', + port: 0, + accountCreated: RANGE_START, + logRequests: false, + validateQueryParams: true, + }) + try { + const res = await fetch(`${validatingServer.url}/v1/customers?foo=bar`, { + headers: { Authorization: 'Bearer sk_test_fake' }, + }) + expect(res.status).toBe(400) + const errBody = (await res.json()) as { + error: { type: string; message: string; details: string[]; allowed: string[] } + } + expect(errBody.error.type).toBe('invalid_request_error') + expect(errBody.error.message).toBe('Query parameters do not match OpenAPI definition') + expect(errBody.error.details).toContain('Unknown query parameter "foo"') + expect(errBody.error.allowed).toContain('limit') + } finally { + await validatingServer.close().catch(() => {}) + } + }, 120_000) + + for (const supportedApiVersion of SUPPORTED_API_VERSIONS) { + it( + `syncs all supported streams for Stripe API ${supportedApiVersion}`, + async () => { + const year = parseInt(supportedApiVersion.slice(0, 4), 10) + if (year < 2020) { + return + } + await syncAllEndpointsForVersion(supportedApiVersion) + }, + 3 * 60_000 + ) + } +}) diff --git a/e2e/test-server-harness.ts b/e2e/test-server-harness.ts new file mode 100644 index 000000000..5dcefa2bc --- /dev/null +++ b/e2e/test-server-harness.ts @@ -0,0 +1,286 @@ +import { execSync } from 'node:child_process' +import path from 'node:path' +import pg from 'pg' +import { + applyCreatedTimestampRange, + createStripeListServer, + ensureObjectTable, + ensureSchema, + startDockerPostgres18, + upsertObjects, + type DockerPostgres18Handle, + type StripeListServer, + type StripeListServerOptions, +} from '@stripe/sync-test-utils' +import { + BUNDLED_API_VERSION, + generateObjectsFromSchema, + resolveOpenApiSpec, +} from '@stripe/sync-openapi' + +export const SERVICE_URL = process.env.SERVICE_URL ?? 'http://localhost:4020' +export const ENGINE_URL = process.env.ENGINE_URL ?? 'http://localhost:4010' +export const CONTAINER_HOST = process.env.CONTAINER_HOST ?? 'host.docker.internal' +export const SKIP_SETUP = process.env.SKIP_SETUP === '1' +export const REPO_ROOT = path.resolve(import.meta.dirname, '..') +export const COMPOSE_CMD = `docker compose -f compose.yml -f compose.dev.yml -f e2e/compose.e2e.yml` + +export const CUSTOMER_COUNT = 10_000 +export const SEED_BATCH = 1000 +export const SOURCE_SCHEMA = 'stripe' + +export function utc(date: string): number { + return Math.floor(new Date(date + 'T00:00:00Z').getTime() / 1000) +} + +export const RANGE_START = utc('2021-04-03') +export const RANGE_END = utc('2026-04-02') + +export type StartServiceHarnessOptions = { + customerCount?: number + seedBatchSize?: number + listServer?: Partial> +} + +export async function pollUntil( + fn: () => Promise, + { timeout = 300_000, interval = 2000 } = {} +): Promise { + const deadline = Date.now() + timeout + while (Date.now() < deadline) { + if (await fn()) return + await new Promise((resolve) => setTimeout(resolve, interval)) + } + throw new Error(`pollUntil timed out after ${timeout}ms`) +} + +async function isServiceHealthy(): Promise { + try { + const res = await fetch(`${SERVICE_URL}/health`) + return res.ok + } catch { + return false + } +} + +async function isEngineHealthy(): Promise { + try { + const res = await fetch(`${ENGINE_URL}/health`) + return res.ok + } catch { + return false + } +} + +async function ensureDockerStack(): Promise { + console.log('\n Building packages...') + execSync('pnpm build', { cwd: REPO_ROOT, stdio: 'inherit' }) + console.log(' Starting Docker stack...') + execSync(`${COMPOSE_CMD} up --build -d temporal engine service worker`, { + cwd: REPO_ROOT, + stdio: 'inherit', + }) + console.log(' Waiting for service health...') + await pollUntil(isServiceHealthy, { timeout: 180_000 }) +} + +export async function ensureServiceStack(): Promise { + if (!SKIP_SETUP) { + await ensureDockerStack() + } + await pollUntil(isServiceHealthy, { timeout: 60_000 }) +} + +export async function ensureEngineStack(): Promise { + if (!SKIP_SETUP) { + await ensureDockerStack() + } + await pollUntil(isEngineHealthy, { timeout: 60_000 }) +} + +function pool(connectionString: string): pg.Pool { + const next = new pg.Pool({ connectionString }) + next.on('error', () => {}) + return next +} + +function dockerOutput(command: string): string { + return execSync(command, { cwd: REPO_ROOT, encoding: 'utf8' }).trim() +} + +export function composeContainerId(serviceName: string): string { + const containerId = dockerOutput(`${COMPOSE_CMD} ps -q ${serviceName}`) + if (!containerId) { + throw new Error(`No running container found for compose service "${serviceName}"`) + } + return containerId +} + +export function pauseDockerContainer(containerId: string): void { + execSync(`docker pause ${containerId}`, { cwd: REPO_ROOT, stdio: 'pipe' }) +} + +export function unpauseDockerContainer(containerId: string): void { + execSync(`docker unpause ${containerId}`, { cwd: REPO_ROOT, stdio: 'pipe' }) +} + +export function pauseComposeService(serviceName: string): string { + const containerId = composeContainerId(serviceName) + pauseDockerContainer(containerId) + return containerId +} + +export function unpauseComposeService(serviceName: string): string { + const containerId = composeContainerId(serviceName) + unpauseDockerContainer(containerId) + return containerId +} + +async function loadBundledSpec() { + return (await resolveOpenApiSpec({ apiVersion: BUNDLED_API_VERSION }, fetch)).spec +} + +function generateTemplate( + spec: import('@stripe/sync-openapi').OpenApiSpec, + schemaName: string, + tableName?: string +): Record { + return generateObjectsFromSchema(spec, schemaName, 1, { tableName })[0] +} + +export type ServiceHarness = { + sourceDocker: DockerPostgres18Handle + destDocker: DockerPostgres18Handle + destPool: pg.Pool + testServer: StripeListServer + expectedIds: string[] + testServerContainerUrl: () => string + destPgContainerUrl: () => string + close: () => Promise +} + +export async function startServiceHarness( + options: StartServiceHarnessOptions = {} +): Promise { + await ensureServiceStack() + + const [sourceDocker, destDocker, spec] = await Promise.all([ + startDockerPostgres18(), + startDockerPostgres18(), + loadBundledSpec(), + ]) + const sourcePool = pool(sourceDocker.connectionString) + const destPool = pool(destDocker.connectionString) + + await ensureSchema(sourcePool, SOURCE_SCHEMA) + await ensureObjectTable(sourcePool, SOURCE_SCHEMA, 'customers') + + const count = options.customerCount ?? CUSTOMER_COUNT + const batchSize = options.seedBatchSize ?? SEED_BATCH + const template = generateTemplate(spec, 'customer', 'customers') + const objects = applyCreatedTimestampRange( + Array.from({ length: count }, (_, i) => ({ + ...template, + id: `cus_test_${String(i).padStart(5, '0')}`, + created: 0, + })), + { startUnix: RANGE_START, endUnix: RANGE_END } + ) + for (let i = 0; i < objects.length; i += batchSize) { + await upsertObjects(sourcePool, SOURCE_SCHEMA, 'customers', objects.slice(i, i + batchSize)) + } + const expectedIds = objects.map((o) => o.id as string) + + const testServer = await createStripeListServer({ + ...options.listServer, + postgresUrl: sourceDocker.connectionString, + host: '0.0.0.0', + port: 0, + accountCreated: options.listServer?.accountCreated ?? RANGE_START, + }) + + console.log(` Source PG: ${sourceDocker.connectionString}`) + console.log(` Dest PG: ${destDocker.connectionString}`) + console.log(` Test server: http://0.0.0.0:${testServer.port}`) + console.log(` Service API: ${SERVICE_URL}`) + console.log(` Container host: ${CONTAINER_HOST}`) + + return { + sourceDocker, + destDocker, + destPool, + testServer, + expectedIds, + testServerContainerUrl: () => `http://${CONTAINER_HOST}:${testServer.port}`, + destPgContainerUrl: () => destDocker.connectionString.replace('localhost', CONTAINER_HOST), + close: async () => { + await testServer.close().catch(() => {}) + await sourcePool.end().catch(() => {}) + await destPool.end().catch(() => {}) + await destDocker.stop() + await sourceDocker.stop() + }, + } +} + +export type EngineHarness = { + sourceDocker: DockerPostgres18Handle + destDocker: DockerPostgres18Handle + testServer: StripeListServer + sourcePool: pg.Pool + destPool: pg.Pool + customerTemplate: Record + productTemplate: Record + hostTestServerUrl: () => string + testServerContainerUrl: () => string + destPgContainerUrl: () => string + close: () => Promise +} + +export async function startEngineHarness(): Promise { + await ensureEngineStack() + + const [sourceDocker, destDocker, spec] = await Promise.all([ + startDockerPostgres18(), + startDockerPostgres18(), + loadBundledSpec(), + ]) + const customerTemplate = generateTemplate(spec, 'customer', 'customers') + const productTemplate = generateTemplate(spec, 'product', 'products') + + const sourcePool = pool(sourceDocker.connectionString) + const destPool = pool(destDocker.connectionString) + + await ensureSchema(sourcePool, SOURCE_SCHEMA) + await Promise.all([ + ensureObjectTable(sourcePool, SOURCE_SCHEMA, 'customers'), + ensureObjectTable(sourcePool, SOURCE_SCHEMA, 'products'), + ]) + + const testServer = await createStripeListServer({ + postgresUrl: sourceDocker.connectionString, + host: '0.0.0.0', + port: 0, + accountCreated: RANGE_START, + }) + + return { + sourceDocker, + destDocker, + testServer, + sourcePool, + destPool, + customerTemplate, + productTemplate, + hostTestServerUrl: () => `http://127.0.0.1:${testServer.port}`, + testServerContainerUrl: () => `http://${CONTAINER_HOST}:${testServer.port}`, + destPgContainerUrl: () => destDocker.connectionString.replace('localhost', CONTAINER_HOST), + close: async () => { + await testServer.close().catch(() => {}) + await sourcePool.end().catch(() => {}) + await destPool.end().catch(() => {}) + await destDocker.stop() + await sourceDocker.stop() + }, + } +} diff --git a/e2e/test-server-sync.test.ts b/e2e/test-server-sync.test.ts new file mode 100644 index 000000000..53a64eded --- /dev/null +++ b/e2e/test-server-sync.test.ts @@ -0,0 +1,697 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + applyCreatedTimestampRange, + ensureObjectTable, + quoteIdentifier, + upsertObjects, +} from '@stripe/sync-test-utils' +import { + createRemoteEngine, + type Message, + type PipelineConfig, + type SourceState, + type SyncOutput, +} from '@stripe/sync-engine' +import { expandState, type BackfillState, type StripeStreamState } from '@stripe/sync-source-stripe' +import { + ENGINE_URL, + RANGE_END, + RANGE_START, + SEED_BATCH, + SOURCE_SCHEMA, + startEngineHarness, + type EngineHarness, +} from './test-server-harness.js' + +describe('test-server sync via Docker engine', () => { + const engine = createRemoteEngine(ENGINE_URL) + const createdSchemas: string[] = [] + let harness: EngineHarness + let schemaCounter = 0 + + function uniqueSchema(prefix: string): string { + const name = `${prefix}_${Date.now()}_${schemaCounter++}` + createdSchemas.push(name) + return name + } + + function makeCustomer(id: string, created: number): Record { + return { ...harness.customerTemplate, id, created } + } + + function makeProduct(id: string, created: number): Record { + return { ...harness.productTemplate, id, created } + } + + function mkBackfill(overrides: Partial = {}): BackfillState { + return { + range: { gte: RANGE_START, lt: RANGE_END }, + num_segments: 5, + completed: [], + in_flight: [], + ...overrides, + } + } + + function pendingState(overrides: Partial = {}): StripeStreamState { + return { + page_cursor: null, + status: 'pending', + backfill: mkBackfill(overrides), + } + } + + function completeState(overrides: Partial = {}): StripeStreamState { + return { + page_cursor: null, + status: 'complete', + backfill: { + range: { gte: RANGE_START, lt: RANGE_END }, + num_segments: 5, + completed: [{ gte: RANGE_START, lt: RANGE_END }], + in_flight: [], + ...overrides, + }, + } + } + + function sourceState(streams: Record): SourceState { + return { streams, global: {} } + } + + function cloneSourceState(initial?: SourceState): SourceState { + return { + streams: { ...initial?.streams }, + global: { ...initial?.global }, + } + } + + function captureSourceState( + state: SourceState, + msg: { + source_state: { + state_type?: 'stream' | 'global' + stream?: string + data: unknown + } + } + ): void { + if (msg.source_state.state_type === 'global') { + state.global = msg.source_state.data as Record + return + } + if (msg.source_state.stream) { + state.streams[msg.source_state.stream] = msg.source_state.data as Record + } + } + + async function batchUpsert(table: string, objects: Record[]) { + for (let i = 0; i < objects.length; i += SEED_BATCH) { + await upsertObjects( + harness.sourcePool, + SOURCE_SCHEMA, + table, + objects.slice(i, i + SEED_BATCH) + ) + } + } + + async function replaceTableObjects(table: string, objects: Record[]) { + await ensureObjectTable(harness.sourcePool, SOURCE_SCHEMA, table) + await harness.sourcePool.query( + `TRUNCATE TABLE ${quoteIdentifier(SOURCE_SCHEMA)}.${quoteIdentifier(table)}` + ) + if (objects.length > 0) { + await batchUpsert(table, objects) + } + } + + async function seedCustomers(objects: Record[]) { + await replaceTableObjects('customers', objects) + } + + function generateCustomers(count: number, prefix: string): Record[] { + const shells = Array.from({ length: count }, (_, i) => + makeCustomer(`${prefix}${String(i).padStart(5, '0')}`, 0) + ) + return applyCreatedTimestampRange(shells, { startUnix: RANGE_START, endUnix: RANGE_END }) + } + + function makePipelineConfig(opts: { + destSchema: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + }): PipelineConfig { + return { + source: { + type: 'stripe', + stripe: { + api_key: 'sk_test_fake', + api_version: '2025-04-30.basil', + base_url: harness.testServerContainerUrl(), + rate_limit: 10_000, + ...opts.sourceOverrides, + }, + }, + destination: { + type: 'postgres', + postgres: { + connection_string: harness.destPgContainerUrl(), + schema: opts.destSchema, + batch_size: 100, + }, + }, + streams: opts.streams ?? [{ name: 'customers', sync_mode: 'full_refresh' }], + } + } + + async function runRead(opts: { + destSchema: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + state?: SourceState + state_limit?: number + time_limit?: number + }): Promise<{ messages: Message[]; state: SourceState }> { + const pipeline = makePipelineConfig(opts) + const messages: Message[] = [] + const state = cloneSourceState(opts.state) + + for await (const msg of engine.pipeline_read(pipeline, { + state: opts.state, + state_limit: opts.state_limit, + time_limit: opts.time_limit, + })) { + messages.push(msg) + if (msg.type === 'source_state') { + captureSourceState(state, msg) + } + } + + return { messages, state } + } + + async function runSync(opts: { + destSchema: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + state?: SourceState + state_limit?: number + time_limit?: number + }): Promise<{ messages: SyncOutput[]; state: SourceState }> { + const pipeline = makePipelineConfig(opts) + const messages: SyncOutput[] = [] + const state = cloneSourceState(opts.state) + + for await (const setupMsg of engine.pipeline_setup(pipeline)) { + // Destination Postgres needs setup to create schema/table structures. + void setupMsg + } + + for await (const msg of engine.pipeline_sync(pipeline, { + state: opts.state, + state_limit: opts.state_limit, + time_limit: opts.time_limit, + })) { + messages.push(msg) + if (msg.type === 'source_state') { + captureSourceState(state, msg) + } + } + + return { messages, state } + } + + async function countRows(schema: string, table: string): Promise { + try { + const { rows } = await harness.destPool.query<{ c: number }>( + `SELECT count(*)::int AS c FROM "${schema}"."${table}"` + ) + return rows[0]?.c ?? 0 + } catch (err) { + if ((err as { code?: string })?.code === '42P01') return 0 + throw err + } + } + + async function listIds(schema: string, table: string): Promise { + try { + const { rows } = await harness.destPool.query<{ id: string }>( + `SELECT id FROM "${schema}"."${table}" ORDER BY id` + ) + return rows.map((row) => row.id) + } catch (err) { + if ((err as { code?: string })?.code === '42P01') return [] + throw err + } + } + + beforeAll(async () => { + harness = await startEngineHarness() + }, 10 * 60_000) + + afterAll(async () => { + for (const schema of createdSchemas) { + await harness?.destPool?.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => {}) + } + await harness?.close() + }, 60_000) + + it('created filter boundaries: objects at segment edges are not lost or duplicated', async () => { + const CONC = 5 + const destSchema = uniqueSchema('boundary') + const segments = expandState(mkBackfill({ num_segments: CONC })) + const internalBoundaries = segments.slice(0, -1).map((segment) => segment.lt) + + const boundaryCustomers = internalBoundaries.flatMap((boundary, i) => [ + makeCustomer(`cus_b${i}_at`, boundary), + makeCustomer(`cus_b${i}_minus1`, boundary - 1), + makeCustomer(`cus_b${i}_plus1`, boundary + 1), + ]) + const edgeCustomers = [ + makeCustomer('cus_range_start', RANGE_START), + makeCustomer('cus_range_start_p1', RANGE_START + 1), + makeCustomer('cus_range_end_m1', RANGE_END - 1), + ] + const expected = [ + ...boundaryCustomers, + ...edgeCustomers, + ...generateCustomers(10_000 - boundaryCustomers.length - edgeCustomers.length, 'cus_bfill_'), + ] + + await seedCustomers(expected) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC }, + state: sourceState({ customers: pendingState({ num_segments: CONC }) }), + }) + + const destIds = new Set(await listIds(destSchema, 'customers')) + for (const customer of expected) { + expect( + destIds.has(customer.id as string), + `missing ${customer.id} (created=${customer.created})` + ).toBe(true) + } + expect(destIds.size).toBe(expected.length) + + const finalState = state.streams.customers as StripeStreamState + expect(finalState.backfill?.range).toEqual({ gte: RANGE_START, lt: RANGE_END }) + expect(finalState.backfill?.num_segments).toBe(CONC) + }, 120_000) + + it('out-of-range objects are excluded by created filter', async () => { + const destSchema = uniqueSchema('outofrange') + const namedInRange = [ + makeCustomer('cus_in_start', RANGE_START), + makeCustomer('cus_in_mid', RANGE_START + 1000), + makeCustomer('cus_in_end_m1', RANGE_END - 1), + ] + const inRange = [...namedInRange, ...generateCustomers(10_000, 'cus_oor_')] + const outOfRange = [ + makeCustomer('cus_out_before_far', RANGE_START - 100), + makeCustomer('cus_out_before_1', RANGE_START - 1), + makeCustomer('cus_out_at_end', RANGE_END), + makeCustomer('cus_out_after_far', RANGE_END + 100), + ] + + await seedCustomers([...inRange, ...outOfRange]) + await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + state: sourceState({ customers: pendingState({ num_segments: 1 }) }), + }) + + const ids = new Set(await listIds(destSchema, 'customers')) + for (const customer of inRange) { + expect(ids.has(customer.id as string), `expected in-range ${customer.id}`).toBe(true) + } + for (const customer of outOfRange) { + expect(ids.has(customer.id as string), `unexpected out-of-range ${customer.id}`).toBe(false) + } + expect(ids.size).toBe(inRange.length) + }, 120_000) + + it('multi-page: >100 objects in a segment forces pagination', async () => { + const destSchema = uniqueSchema('multipage') + const COUNT = 10_000 + + await seedCustomers(generateCustomers(COUNT, 'cus_mp_')) + + const { messages } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + expect(await countRows(destSchema, 'customers')).toBe(COUNT) + expect(messages.filter((msg) => msg.type === 'source_state').length).toBeGreaterThan(1) + }, 120_000) + + it('no duplicate record IDs emitted by source across segments', async () => { + const CONC = 5 + const destSchema = uniqueSchema('dupcheck') + const segments = expandState(mkBackfill({ num_segments: CONC })) + const boundaries = segments.slice(0, -1).map((segment) => segment.lt) + + const boundaryObjects = boundaries.flatMap((boundary, i) => [ + makeCustomer(`cus_d${i}_at`, boundary), + makeCustomer(`cus_d${i}_m1`, boundary - 1), + makeCustomer(`cus_d${i}_p1`, boundary + 1), + ]) + boundaryObjects.push(makeCustomer('cus_d_start', RANGE_START)) + boundaryObjects.push(makeCustomer('cus_d_end_m1', RANGE_END - 1)) + + const objects = [ + ...boundaryObjects, + ...generateCustomers(10_000 - boundaryObjects.length, 'cus_dfill_'), + ] + + await seedCustomers(objects) + + const { messages } = await runRead({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC }, + state: sourceState({ customers: pendingState({ num_segments: CONC }) }), + }) + + const recordIds = messages + .filter((msg) => msg.type === 'record') + .map((msg) => msg.record.data.id) + .filter((id): id is string => typeof id === 'string') + + expect(recordIds.length, 'source emitted duplicate record IDs').toBe(new Set(recordIds).size) + expect(recordIds.length).toBe(objects.length) + }, 120_000) + + it('resume from partially-completed state skips completed segments', async () => { + const destSchema = uniqueSchema('resume') + const CONC = 5 + const segments = expandState(mkBackfill({ num_segments: CONC })) + const PER_SEGMENT = 2000 + + const objects = segments.flatMap((segment, segIdx) => { + const step = Math.max(1, Math.floor((segment.lt - segment.gte - 2) / PER_SEGMENT)) + return Array.from({ length: PER_SEGMENT }, (_, i) => + makeCustomer(`cus_seg${segIdx}_${String(i).padStart(4, '0')}`, segment.gte + 1 + i * step) + ) + }) + + await seedCustomers(objects) + + const completedRange = { gte: segments[0].gte, lt: segments[2].lt } + await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC }, + state: sourceState({ + customers: pendingState({ + num_segments: CONC, + completed: [completedRange], + }), + }), + }) + + const destIds = new Set(await listIds(destSchema, 'customers')) + for (const segIdx of [3, 4]) { + for (let i = 0; i < PER_SEGMENT; i++) { + const id = `cus_seg${segIdx}_${String(i).padStart(4, '0')}` + expect(destIds.has(id), `missing ${id}`).toBe(true) + } + } + for (const segIdx of [0, 1, 2]) { + expect(destIds.has(`cus_seg${segIdx}_0000`), `unexpected cus_seg${segIdx}_0000`).toBe(false) + } + expect(destIds.size).toBe(PER_SEGMENT * 2) + }, 120_000) + + it('empty segments complete without hanging', async () => { + const destSchema = uniqueSchema('empty') + const CONC = 5 + const segments = expandState(mkBackfill({ num_segments: CONC })) + const populatedSegments = [0, 2, 4] + const perSegment = Math.ceil(10_000 / populatedSegments.length) + + const objects = populatedSegments.flatMap((segIdx) => { + const segment = segments[segIdx] + const step = Math.max(1, Math.floor((segment.lt - segment.gte - 2) / perSegment)) + return Array.from({ length: perSegment }, (_, i) => + makeCustomer(`cus_e${segIdx}_${String(i).padStart(4, '0')}`, segment.gte + 1 + i * step) + ) + }) + + await seedCustomers(objects) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC }, + state: sourceState({ customers: pendingState({ num_segments: CONC }) }), + }) + + expect(await countRows(destSchema, 'customers')).toBe(objects.length) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + }, 120_000) + + it('second sync after completion emits zero records', async () => { + const destSchema = uniqueSchema('idempotent') + + await seedCustomers(generateCustomers(10_000, 'cus_idem_')) + + const { messages } = await runSync({ + destSchema, + state: sourceState({ customers: completeState() }), + }) + + expect(messages.filter((msg) => msg.type === 'source_state').length).toBe(0) + expect(await countRows(destSchema, 'customers')).toBe(0) + }, 120_000) + + it('backfill_limit stops fetching after the threshold', async () => { + const destSchema = uniqueSchema('bflimit') + const TOTAL = 10_000 + + await seedCustomers(generateCustomers(TOTAL, 'cus_bl_')) + + const { messages } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + streams: [{ name: 'customers', sync_mode: 'full_refresh', backfill_limit: 5 }], + }) + + const synced = await countRows(destSchema, 'customers') + expect(synced).toBeGreaterThan(0) + expect(synced).toBeLessThan(TOTAL) + expect(messages.filter((msg) => msg.type === 'source_state').length).toBeGreaterThan(0) + }, 120_000) + + it('pagination handles ID/created order mismatch correctly', async () => { + const destSchema = uniqueSchema('idorder') + const COUNT = 10_000 + const timestamps = Array.from({ length: 5 }, (_, i) => RANGE_START + (i + 1) * 1000) + const objects = Array.from({ length: COUNT }, (_, i) => + makeCustomer(`cus_tie_${String(i).padStart(5, '0')}`, timestamps[i % timestamps.length]!) + ) + + await seedCustomers(objects) + await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + const destIds = new Set(await listIds(destSchema, 'customers')) + for (const object of objects) { + expect(destIds.has(object.id as string), `missing ${object.id}`).toBe(true) + } + expect(destIds.size).toBe(COUNT) + }, 120_000) + + it('syncs multiple streams in a single run', async () => { + const destSchema = uniqueSchema('multistream') + const PER_STREAM = 5000 + const range = { startUnix: RANGE_START, endUnix: RANGE_END } + + const customers = applyCreatedTimestampRange( + Array.from({ length: PER_STREAM }, (_, i) => + makeCustomer(`cus_ms_${String(i).padStart(5, '0')}`, 0) + ), + range + ) + const products = applyCreatedTimestampRange( + Array.from({ length: PER_STREAM }, (_, i) => + makeProduct(`prod_ms_${String(i).padStart(5, '0')}`, 0) + ), + range + ) + + await Promise.all([ + replaceTableObjects('customers', customers), + replaceTableObjects('products', products), + ]) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + streams: [ + { name: 'customers', sync_mode: 'full_refresh' }, + { name: 'products', sync_mode: 'full_refresh' }, + ], + }) + + expect(await countRows(destSchema, 'customers')).toBe(customers.length) + expect(await countRows(destSchema, 'products')).toBe(products.length) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + expect((state.streams.products as StripeStreamState).status).toBe('complete') + }, 120_000) + + it('zero objects: empty source completes cleanly with no records', async () => { + const destSchema = uniqueSchema('zerobj') + + await seedCustomers([]) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + expect(await countRows(destSchema, 'customers')).toBe(0) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + }, 120_000) + + it('single object: exactly one record syncs correctly', async () => { + const destSchema = uniqueSchema('single') + + await seedCustomers([makeCustomer('cus_only_one', RANGE_START + 500)]) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + const ids = await listIds(destSchema, 'customers') + expect(ids).toEqual(['cus_only_one']) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + }, 120_000) + + it('data integrity: destination _raw_data matches source objects', async () => { + const destSchema = uniqueSchema('integrity') + const sourceObjects = generateCustomers(10_000, 'cus_int_') + + await seedCustomers(sourceObjects) + await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + expect(await countRows(destSchema, 'customers')).toBe(sourceObjects.length) + + const sample = [sourceObjects[0], sourceObjects[4999], sourceObjects[9999]] + for (const object of sample) { + const { rows } = await harness.destPool.query<{ _raw_data: Record }>( + `SELECT "_raw_data" FROM "${destSchema}"."customers" WHERE id = $1`, + [object!.id] + ) + expect(rows.length, `missing ${object!.id} in destination`).toBe(1) + const dest = rows[0]!._raw_data + expect(dest.id).toBe(object!.id) + expect(dest.created).toBe(object!.created) + expect(dest.object).toBe('customer') + expect(dest.email).toBe(object!.email) + } + }, 120_000) + + it('multi-page pagination across multiple concurrent segments', async () => { + const destSchema = uniqueSchema('multipageseg') + const CONC = 3 + const segments = expandState(mkBackfill({ num_segments: CONC })) + const PER_SEGMENT = 3334 + + const objects = segments.flatMap((segment, segIdx) => { + const step = Math.max(1, Math.floor((segment.lt - segment.gte - 2) / PER_SEGMENT)) + return Array.from({ length: PER_SEGMENT }, (_, i) => + makeCustomer(`cus_mps${segIdx}_${String(i).padStart(4, '0')}`, segment.gte + 1 + i * step) + ) + }) + + await seedCustomers(objects) + + const { messages, state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC }, + state: sourceState({ customers: pendingState({ num_segments: CONC }) }), + }) + + expect(await countRows(destSchema, 'customers')).toBe(objects.length) + expect(messages.filter((msg) => msg.type === 'source_state').length).toBeGreaterThan(CONC) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + }, 120_000) + + it('stress: 200 segments with 100k objects at 1000 req/s', async () => { + const destSchema = uniqueSchema('stress') + const CONC = 200 + const TOTAL = 100_000 + const segments = expandState(mkBackfill({ num_segments: CONC })) + const perSegment = Math.ceil(TOTAL / segments.length) + const objects: Record[] = [] + + for (let segIdx = 0; segIdx < segments.length; segIdx++) { + const segment = segments[segIdx]! + const step = Math.max(1, Math.floor((segment.lt - segment.gte - 2) / perSegment)) + for (let i = 0; i < perSegment && objects.length < TOTAL; i++) { + objects.push( + makeCustomer( + `cus_s_${String(objects.length).padStart(6, '0')}`, + segment.gte + 1 + i * step + ) + ) + } + } + + await seedCustomers(objects) + + const { state } = await runSync({ + destSchema, + sourceOverrides: { backfill_concurrency: CONC, rate_limit: 1_000 }, + state: sourceState({ customers: pendingState({ num_segments: CONC }) }), + }) + + const destIds = new Set(await listIds(destSchema, 'customers')) + const expectedIds = new Set(objects.map((object) => object.id as string)) + const missing = [...expectedIds].filter((id) => !destIds.has(id)) + const unexpected = [...destIds].filter((id) => !expectedIds.has(id)) + + expect( + missing.length, + `missing ${missing.length} objects, first 10: ${missing.slice(0, 10).join(', ')}` + ).toBe(0) + expect(unexpected.length, `unexpected ${unexpected.length} objects`).toBe(0) + expect(destIds.size).toBe(TOTAL) + expect((state.streams.customers as StripeStreamState).status).toBe('complete') + }, 600_000) + + it('v2 stream: syncs v2_core_event_destinations via cursor pagination', async () => { + const destSchema = uniqueSchema('v2sync') + const STREAM = 'v2_core_event_destinations' + + const v2Objects = Array.from({ length: 10_000 }, (_, i) => ({ + id: `ed_test_${String(i).padStart(5, '0')}`, + object: 'v2.core.event_destination', + description: `Event destination ${i}`, + status: 'enabled', + enabled_events: ['*'], + metadata: {}, + })) + + await replaceTableObjects(STREAM, v2Objects) + + const { state } = await runSync({ + destSchema, + streams: [{ name: STREAM, sync_mode: 'full_refresh' }], + }) + + const destIds = new Set(await listIds(destSchema, STREAM)) + for (const object of v2Objects) { + expect(destIds.has(object.id), `missing v2 object ${object.id}`).toBe(true) + } + expect(destIds.size).toBe(v2Objects.length) + expect((state.streams[STREAM] as StripeStreamState).status).toBe('complete') + }, 120_000) +}) diff --git a/e2e/test-sync-e2e.test.ts b/e2e/test-sync-e2e.test.ts new file mode 100644 index 000000000..654679f71 --- /dev/null +++ b/e2e/test-sync-e2e.test.ts @@ -0,0 +1,81 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + SERVICE_URL, + pollUntil, + startServiceHarness, + type ServiceHarness, +} from './test-server-harness.js' + +describe('test-server sync via Docker service: 10k customers', () => { + let harness: ServiceHarness + + beforeAll(async () => { + harness = await startServiceHarness() + expect(harness.expectedIds.length).toBeGreaterThan(0) + }, 10 * 60_000) + + afterAll(async () => { + await harness?.close() + }, 60_000) + + it( + 'POST /pipelines syncs 10k customers from test server to Postgres', + async () => { + const destSchema = `e2e_server_sync_${Date.now()}` + + const createRes = await fetch(`${SERVICE_URL}/pipelines`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + source: { + type: 'stripe', + stripe: { + api_key: 'sk_test_fake', + base_url: harness.testServerContainerUrl(), + rate_limit: 1000, + }, + }, + destination: { + type: 'postgres', + postgres: { + connection_string: harness.destPgContainerUrl(), + schema: destSchema, + }, + }, + streams: [{ name: 'customers' }], + }), + }) + expect(createRes.status).toBe(201) + + const created = (await createRes.json()) as { id: string } + const id = created.id + expect(id).toMatch(/^pipe_/) + + await pollUntil(async () => { + try { + const r = await harness.destPool.query( + `SELECT count(*)::int AS n FROM "${destSchema}"."customers"` + ) + return r.rows[0].n === harness.expectedIds.length + } catch { + return false + } + }) + + const { rows } = await harness.destPool.query( + `SELECT id FROM "${destSchema}"."customers" ORDER BY id` + ) + const destIds = new Set(rows.map((r: { id: string }) => r.id)) + expect(destIds.size).toBe(harness.expectedIds.length) + for (const expectedId of harness.expectedIds) { + expect(destIds.has(expectedId), `missing ${expectedId}`).toBe(true) + } + + const delRes = await fetch(`${SERVICE_URL}/pipelines/${id}`, { method: 'DELETE' }) + expect(delRes.status).toBe(200) + + await harness.destPool.query(`DROP SCHEMA IF EXISTS "${destSchema}" CASCADE`) + }, + 15 * 60_000 + ) +}) diff --git a/e2e/test-sync-engine.test.ts b/e2e/test-sync-engine.test.ts new file mode 100644 index 000000000..d5ba37fbf --- /dev/null +++ b/e2e/test-sync-engine.test.ts @@ -0,0 +1,373 @@ +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from 'vitest' +import { ensureObjectTable, quoteIdentifier, upsertObjects } from '@stripe/sync-test-utils' +import { + createEngine, + type ConnectorResolver, + type Engine, + type Message, + type PipelineConfig, + type SourceState, + type SyncOutput, +} from '../apps/engine/src/index.js' +import stripeSource from '../packages/source-stripe/src/index.js' +import postgresDestination from '../packages/destination-postgres/src/index.js' +import { + RANGE_START, + SOURCE_SCHEMA, + startEngineHarness, + type EngineHarness, +} from './test-server-harness.js' +import { BUNDLED_API_VERSION } from '../packages/openapi/src/versions.js' +import { createStripeListServer } from '../packages/test-utils/src/server/createStripeListServer.js' +import type { + StripeListServer, + StripeListServerOptions, +} from '../packages/test-utils/src/server/types.js' + +describe('Stripe failure handling via Docker engine', () => { + const createdSchemas: string[] = [] + const injectedServers: StripeListServer[] = [] + let harness: EngineHarness + let engine: Engine + let schemaCounter = 0 + + function makeResolver(): ConnectorResolver { + return { + resolveSource: async (name) => { + if (name !== 'stripe') throw new Error(`Unknown source: ${name}`) + return stripeSource + }, + resolveDestination: async (name) => { + if (name !== 'postgres') throw new Error(`Unknown destination: ${name}`) + return postgresDestination + }, + sources: () => new Map(), + destinations: () => new Map(), + } + } + + function uniqueSchema(prefix: string): string { + const name = `${prefix}_${Date.now()}_${schemaCounter++}` + createdSchemas.push(name) + return name + } + + function makeCustomer(id: string, created: number): Record { + return { ...harness.customerTemplate, id, created } + } + + function makeProduct(id: string, created: number): Record { + return { ...harness.productTemplate, id, created } + } + + function generateCustomers(count: number, prefix: string): Record[] { + return Array.from({ length: count }, (_, index) => + makeCustomer(`${prefix}${index.toString().padStart(3, '0')}`, RANGE_START + index + 1) + ) + } + + function cloneSourceState(initial?: SourceState): SourceState { + return { + streams: { ...initial?.streams }, + global: { ...initial?.global }, + } + } + + function captureSourceState( + state: SourceState, + msg: { + source_state: { + state_type?: 'stream' | 'global' + stream?: string + data: unknown + } + } + ): void { + if (msg.source_state.state_type === 'global') { + state.global = msg.source_state.data as Record + return + } + if (msg.source_state.stream) { + state.streams[msg.source_state.stream] = msg.source_state.data as Record + } + } + + async function batchUpsert(table: string, objects: Record[]) { + for (let i = 0; i < objects.length; i += 1000) { + await upsertObjects(harness.sourcePool, SOURCE_SCHEMA, table, objects.slice(i, i + 1000)) + } + } + + async function replaceTableObjects(table: string, objects: Record[]) { + await ensureObjectTable(harness.sourcePool, SOURCE_SCHEMA, table) + await harness.sourcePool.query( + `TRUNCATE TABLE ${quoteIdentifier(SOURCE_SCHEMA)}.${quoteIdentifier(table)}` + ) + if (objects.length > 0) { + await batchUpsert(table, objects) + } + } + + async function seedCustomers(objects: Record[]) { + await replaceTableObjects('customers', objects) + } + + async function seedProducts(objects: Record[]) { + await replaceTableObjects('products', objects) + } + + async function startInjectedServer( + overrides: Partial> + ): Promise { + const server = await createStripeListServer({ + postgresUrl: harness.sourceDocker.connectionString, + host: '0.0.0.0', + port: 0, + accountCreated: overrides.accountCreated ?? RANGE_START, + auth: overrides.auth, + failures: overrides.failures, + }) + injectedServers.push(server) + return server + } + + function makePipelineConfig(opts: { + destSchema: string + baseUrl: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + }): PipelineConfig { + return { + source: { + type: 'stripe', + stripe: { + api_key: 'sk_test_fake', + api_version: BUNDLED_API_VERSION, + base_url: opts.baseUrl, + rate_limit: 1000, + ...opts.sourceOverrides, + }, + }, + destination: { + type: 'postgres', + postgres: { + connection_string: harness.destDocker.connectionString, + schema: opts.destSchema, + batch_size: 100, + }, + }, + streams: opts.streams ?? [{ name: 'customers', sync_mode: 'full_refresh' }], + } + } + + async function runSync(opts: { + destSchema: string + baseUrl: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + state?: SourceState + }): Promise<{ messages: SyncOutput[]; state: SourceState }> { + const pipeline = makePipelineConfig(opts) + const messages: SyncOutput[] = [] + const state = cloneSourceState(opts.state) + + for await (const setupMsg of engine.pipeline_setup(pipeline)) { + void setupMsg + } + + for await (const msg of engine.pipeline_sync(pipeline, { state: opts.state })) { + messages.push(msg) + if (msg.type === 'source_state') { + captureSourceState(state, msg) + } + } + + return { messages, state } + } + + async function runRead(opts: { + destSchema: string + baseUrl: string + streams?: PipelineConfig['streams'] + sourceOverrides?: Record + state?: SourceState + }): Promise<{ messages: Message[]; state: SourceState }> { + const pipeline = makePipelineConfig(opts) + const messages: Message[] = [] + const state = cloneSourceState(opts.state) + + for await (const msg of engine.pipeline_read(pipeline, { state: opts.state })) { + messages.push(msg) + if (msg.type === 'source_state') { + captureSourceState(state, msg) + } + } + + return { messages, state } + } + + async function countRows(schema: string, table: string): Promise { + try { + const { rows } = await harness.destPool.query<{ c: number }>( + `SELECT count(*)::int AS c FROM "${schema}"."${table}"` + ) + return rows[0]?.c ?? 0 + } catch (err) { + if ((err as { code?: string })?.code === '42P01') return 0 + throw err + } + } + + function getErrorTrace(messages: Array, stream: string) { + return messages.find( + (msg): msg is Extract => + msg.type === 'trace' && + msg.trace.trace_type === 'error' && + (msg.trace as { error: { stream?: string } }).error.stream === stream + ) + } + + beforeAll(async () => { + harness = await startEngineHarness() + engine = await createEngine(makeResolver()) + }, 10 * 60_000) + + beforeEach(async () => { + await Promise.all([seedCustomers([]), seedProducts([])]) + }) + + afterEach(async () => { + while (injectedServers.length > 0) { + await injectedServers + .pop()! + .close() + .catch(() => {}) + } + }) + + afterAll(async () => { + for (const schema of createdSchemas) { + await harness?.destPool?.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`).catch(() => {}) + } + while (injectedServers.length > 0) { + await injectedServers + .pop()! + .close() + .catch(() => {}) + } + await harness?.close() + }, 60_000) + + it('emits Invalid API Key trace before records when account lookup is unauthorized', async () => { + const destSchema = uniqueSchema('sync_invalid_key') + await seedCustomers([makeCustomer('cus_auth_001', RANGE_START + 1)]) + + const server = await startInjectedServer({ + auth: { expectedBearerToken: 'sk_test_valid' }, + }) + + const { messages, state } = await runRead({ + destSchema, + baseUrl: server.url, + sourceOverrides: { + api_key: 'sk_test_bad', + backfill_concurrency: 1, + }, + }) + + const errorTrace = getErrorTrace(messages, 'customers') + expect(errorTrace).toBeDefined() + expect(errorTrace).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'error', + error: { + failure_type: 'system_error', + stream: 'customers', + message: expect.stringContaining('Invalid API Key'), + }, + }, + }) + expect(messages.filter((msg) => msg.type === 'record')).toHaveLength(0) + expect(state.streams.customers).toBeUndefined() + }, 120_000) + + it('continues syncing later streams after one stream returns a non-skippable auth error', async () => { + const destSchema = uniqueSchema('sync_continue_after_error') + await seedCustomers([makeCustomer('cus_fail_001', RANGE_START + 1)]) + await seedProducts([ + makeProduct('prod_ok_001', RANGE_START + 2), + makeProduct('prod_ok_002', RANGE_START + 3), + ]) + + const server = await startInjectedServer({ + failures: [ + { + path: '/v1/customers', + status: 401, + stripeError: { + type: 'invalid_request_error', + message: 'Invalid API Key provided: sk_test_fake', + }, + }, + ], + }) + + const { messages, state } = await runSync({ + destSchema, + baseUrl: server.url, + streams: [ + { name: 'customers', sync_mode: 'full_refresh' }, + { name: 'products', sync_mode: 'full_refresh' }, + ], + sourceOverrides: { backfill_concurrency: 1 }, + }) + + const customerError = getErrorTrace(messages, 'customers') + expect(customerError).toBeDefined() + expect(customerError).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'error', + error: { + failure_type: 'system_error', + stream: 'customers', + message: expect.stringContaining('Invalid API Key'), + }, + }, + }) + expect(await countRows(destSchema, 'customers')).toBe(0) + expect(await countRows(destSchema, 'products')).toBe(2) + expect(state.streams.products).toMatchObject({ status: 'complete' }) + }, 120_000) + + it('retries a later transient pagination failure and completes the stream', async () => { + const destSchema = uniqueSchema('sync_partial_failure') + await seedCustomers(generateCustomers(150, 'cus_partial_')) + + const server = await startInjectedServer({ + failures: [ + { + path: '/v1/customers', + status: 500, + after: 1, + times: 1, + stripeError: { + type: 'api_error', + message: 'Injected page 2 failure', + }, + }, + ], + }) + + const { messages, state } = await runSync({ + destSchema, + baseUrl: server.url, + sourceOverrides: { backfill_concurrency: 1 }, + }) + + expect(getErrorTrace(messages, 'customers')).toBeUndefined() + expect(await countRows(destSchema, 'customers')).toBe(150) + expect(state.streams.customers).toMatchObject({ status: 'complete' }) + }, 120_000) +}) diff --git a/packages/destination-postgres/src/index.ts b/packages/destination-postgres/src/index.ts index 94bb04b3a..4f9ff4775 100644 --- a/packages/destination-postgres/src/index.ts +++ b/packages/destination-postgres/src/index.ts @@ -100,13 +100,22 @@ function isTransient(err: unknown): boolean { return msg.includes('econnrefused') || msg.includes('timeout') || msg.includes('connection') } +function createPool(config: PoolConfig): pg.Pool { + const pool = new pg.Pool(config) + // Destination connectors should surface pool failures without crashing the host process. + pool.on('error', (err) => { + console.error('Postgres destination pool error:', err) + }) + return pool +} + const destination = { async *spec() { yield { type: 'spec' as const, spec: defaultSpec } }, async *check({ config }) { - const pool = new pg.Pool(await buildPoolConfig(config)) + const pool = createPool(await buildPoolConfig(config)) try { await pool.query('SELECT 1') yield { @@ -127,7 +136,7 @@ const destination = { }, async *setup({ config, catalog }) { - const pool = new pg.Pool(await buildPoolConfig(config)) + const pool = createPool(await buildPoolConfig(config)) try { await pool.query(sql`CREATE SCHEMA IF NOT EXISTS "${config.schema}"`) // Ensure the trigger function exists in the target schema so triggers @@ -181,7 +190,7 @@ const destination = { `Refusing to drop protected schema "${config.schema}" — teardown only drops user-created schemas` ) } - const pool = new pg.Pool(await buildPoolConfig(config)) + const pool = createPool(await buildPoolConfig(config)) try { await pool.query(sql`DROP SCHEMA IF EXISTS "${config.schema}" CASCADE`) } finally { @@ -190,7 +199,7 @@ const destination = { }, async *write({ config, catalog }, $stdin) { - const pool = new pg.Pool(await buildPoolConfig(config)) + const pool = createPool(await buildPoolConfig(config)) const batchSize = config.batch_size // eslint-disable-next-line @typescript-eslint/no-explicit-any const streamBuffers = new Map[]>() diff --git a/packages/openapi/__tests__/fixtures/minimalSpec.ts b/packages/openapi/__tests__/fixtures/minimalSpec.ts index 6a8ab5ff3..26a858249 100644 --- a/packages/openapi/__tests__/fixtures/minimalSpec.ts +++ b/packages/openapi/__tests__/fixtures/minimalSpec.ts @@ -10,6 +10,8 @@ function listPath( } if (opts.supportsLimit !== false) { parameters.push({ name: 'limit', in: 'query' }) + parameters.push({ name: 'starting_after', in: 'query' }) + parameters.push({ name: 'ending_before', in: 'query' }) } return { get: { diff --git a/packages/openapi/__tests__/listFnResolver.test.ts b/packages/openapi/__tests__/listFnResolver.test.ts index 693590c9e..ff01ce114 100644 --- a/packages/openapi/__tests__/listFnResolver.test.ts +++ b/packages/openapi/__tests__/listFnResolver.test.ts @@ -12,6 +12,8 @@ describe('discoverListEndpoints', () => { apiPath: '/v1/customers', supportsCreatedFilter: true, supportsLimit: true, + supportsStartingAfter: true, + supportsEndingBefore: true, }) expect(endpoints.get('checkout_sessions')).toEqual({ tableName: 'checkout_sessions', @@ -19,6 +21,8 @@ describe('discoverListEndpoints', () => { apiPath: '/v1/checkout/sessions', supportsCreatedFilter: true, supportsLimit: true, + supportsStartingAfter: true, + supportsEndingBefore: true, }) expect(endpoints.get('early_fraud_warnings')).toEqual({ tableName: 'early_fraud_warnings', @@ -26,6 +30,8 @@ describe('discoverListEndpoints', () => { apiPath: '/v1/radar/early_fraud_warnings', supportsCreatedFilter: true, supportsLimit: true, + supportsStartingAfter: true, + supportsEndingBefore: true, }) }) @@ -38,6 +44,8 @@ describe('discoverListEndpoints', () => { apiPath: '/v2/core/accounts', supportsCreatedFilter: false, supportsLimit: false, + supportsStartingAfter: false, + supportsEndingBefore: false, }) expect(endpoints.get('v2_core_event_destinations')).toEqual({ tableName: 'v2_core_event_destinations', @@ -45,6 +53,8 @@ describe('discoverListEndpoints', () => { apiPath: '/v2/core/event_destinations', supportsCreatedFilter: false, supportsLimit: false, + supportsStartingAfter: false, + supportsEndingBefore: false, }) }) @@ -117,4 +127,95 @@ describe('discoverListEndpoints', () => { expect.anything() ) }) + + it('throws the Stripe error message for non-2xx list responses', async () => { + const fetchMock = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { + type: 'invalid_request_error', + message: 'Invalid API Key provided: sk_test_bad', + }, + }), + { status: 401 } + ) + ) + const list = buildListFn('sk_test_bad', '/v1/customers', fetchMock) + + await expect(list({ limit: 1 })).rejects.toThrow('Invalid API Key provided: sk_test_bad') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('throws for v2 non-2xx list responses', async () => { + const fetchMock = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { type: 'api_error', message: 'Injected page failure' }, + }), + { status: 500 } + ) + ) + const list = buildListFn('sk_test_fake', '/v2/core/accounts', fetchMock) + + await expect(list({ limit: 1 })).rejects.toThrow('Injected page failure') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('encodes created filters for v2 list requests', async () => { + const fetchMock = vi.fn( + async () => + new Response( + JSON.stringify({ + data: [{ id: 'evt_123' }], + next_page_url: '/v2/core/events?page=cur_next&limit=1', + }), + { status: 200 } + ) + ) + const list = buildListFn('sk_test_fake', '/v2/core/events', fetchMock) + + await expect( + list({ + limit: 1, + starting_after: 'cur_prev', + created: { + gte: 1735689600, + lt: 1735776000, + }, + }) + ).resolves.toEqual({ + data: [{ id: 'evt_123' }], + has_more: true, + pageCursor: 'cur_next', + }) + + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/v2/core/events?'), + expect.anything() + ) + + const [url] = fetchMock.mock.calls[0] as [string, RequestInit] + const parsed = new URL(url) + expect(parsed.searchParams.get('limit')).toBe('1') + expect(parsed.searchParams.get('page')).toBe('cur_prev') + expect(parsed.searchParams.get('created[gte]')).toBe('2025-01-01T00:00:00.000Z') + expect(parsed.searchParams.get('created[lt]')).toBe('2025-01-02T00:00:00.000Z') + }) + + it('throws the Stripe error message for non-2xx retrieve responses', async () => { + const fetchMock = vi.fn( + async () => + new Response( + JSON.stringify({ + error: { type: 'invalid_request_error', message: "No such customer: 'cus_missing'" }, + }), + { status: 404 } + ) + ) + const retrieve = buildRetrieveFn('sk_test_fake', '/v1/customers', fetchMock) + + await expect(retrieve('cus_missing')).rejects.toThrow("No such customer: 'cus_missing'") + }) }) diff --git a/packages/openapi/index.ts b/packages/openapi/index.ts index 9ed8605d3..5fbc4f6f9 100644 --- a/packages/openapi/index.ts +++ b/packages/openapi/index.ts @@ -23,6 +23,11 @@ export type { ListParams, } from './listFnResolver.js' export { parsedTableToJsonSchema } from './jsonSchemaConverter.js' +export { + generateObjectsFromSchema, + findSchemaNameByResourceId, +} from './objectGenerator.js' +export type { GenerateObjectsOptions } from './objectGenerator.js' export { StripeAccountSchema, StripeWebhookEndpointSchema, diff --git a/packages/openapi/listFnResolver.ts b/packages/openapi/listFnResolver.ts index 65acabde1..6e0f2a646 100644 --- a/packages/openapi/listFnResolver.ts +++ b/packages/openapi/listFnResolver.ts @@ -22,6 +22,8 @@ export type ListEndpoint = { apiPath: string supportsCreatedFilter: boolean supportsLimit: boolean + supportsStartingAfter: boolean + supportsEndingBefore: boolean } export type NestedEndpoint = { @@ -117,12 +119,20 @@ export function discoverListEndpoints( const supportsLimit = params.some( (p: { name?: string; in?: string }) => p.name === 'limit' && p.in === 'query' ) + const supportsStartingAfter = params.some( + (p: { name?: string; in?: string }) => p.name === 'starting_after' && p.in === 'query' + ) + const supportsEndingBefore = params.some( + (p: { name?: string; in?: string }) => p.name === 'ending_before' && p.in === 'query' + ) endpoints.set(tableName, { tableName, resourceId, apiPath, supportsCreatedFilter, supportsLimit, + supportsStartingAfter, + supportsEndingBefore, }) } } @@ -211,6 +221,51 @@ function authHeaders(apiKey: string): Record { return { Authorization: `Bearer ${apiKey}` } } +class StripeApiRequestError extends Error { + constructor( + public readonly status: number, + public readonly body: unknown, + method: string, + path: string + ) { + super(extractErrorMessage(body, status, method, path)) + this.name = 'StripeApiRequestError' + } +} + +function extractErrorMessage(body: unknown, status: number, method: string, path: string): string { + if ( + body && + typeof body === 'object' && + 'error' in body && + body.error && + typeof body.error === 'object' && + 'message' in body.error && + typeof body.error.message === 'string' + ) { + return body.error.message + } + + return `Stripe API request failed (${status}) for ${method.toUpperCase()} ${path}` +} + +async function readJson(response: Response): Promise { + const text = await response.text() + if (!text) return null + + try { + return JSON.parse(text) as unknown + } catch { + return text + } +} + +function assertOk(response: Response, body: unknown, method: string, path: string): void { + if (!response.ok) { + throw new StripeApiRequestError(response.status, body, method, path) + } +} + /** * Build a callable list function that hits the Stripe HTTP API directly. * Supports both v1 (has_more pagination) and v2 (next_page_url pagination). @@ -229,17 +284,23 @@ export function buildListFn( const qs = new URLSearchParams() qs.set('limit', String(Math.min(params.limit ?? 20, 20))) if (params.starting_after) qs.set('page', params.starting_after) + if (params.created) { + for (const [op, val] of Object.entries(params.created)) { + if (val != null) qs.set(`created[${op}]`, toV2CreatedParam(val)) + } + } const headers = authHeaders(apiKey) if (apiVersion) headers['Stripe-Version'] = apiVersion const response = await fetch(`${base}${apiPath}?${qs}`, { headers }) - const body = (await response.json()) as { + const parsed = (await readJson(response)) as { data: unknown[] next_page_url?: string | null } - const pageCursor = extractPageToken(body.next_page_url) - return { data: body.data ?? [], has_more: !!body.next_page_url, pageCursor } + assertOk(response, parsed, 'GET', apiPath) + const pageCursor = extractPageToken(parsed.next_page_url) + return { data: parsed.data ?? [], has_more: !!parsed.next_page_url, pageCursor } } } @@ -257,11 +318,16 @@ export function buildListFn( const response = await fetch(`${base}${apiPath}?${qs}`, { headers: authHeaders(apiKey), }) - const body = (await response.json()) as { data: unknown[]; has_more: boolean } + const body = (await readJson(response)) as { data: unknown[]; has_more: boolean } + assertOk(response, body, 'GET', apiPath) return { data: body.data ?? [], has_more: body.has_more } } } +function toV2CreatedParam(value: number): string { + return new Date(value * 1000).toISOString() +} + /** * Build a callable retrieve function that hits the Stripe HTTP API directly. */ @@ -280,7 +346,9 @@ export function buildRetrieveFn( if (apiVersion) headers['Stripe-Version'] = apiVersion const response = await fetch(`${base}${apiPath}/${id}`, { headers }) - return await response.json() + const body = await readJson(response) + assertOk(response, body, 'GET', `${apiPath}/${id}`) + return body } } @@ -288,7 +356,9 @@ export function buildRetrieveFn( const response = await fetch(`${base}${apiPath}/${id}`, { headers: authHeaders(apiKey), }) - return await response.json() + const body = await readJson(response) + assertOk(response, body, 'GET', `${apiPath}/${id}`) + return body } } diff --git a/packages/openapi/objectGenerator.ts b/packages/openapi/objectGenerator.ts new file mode 100644 index 000000000..c794a762d --- /dev/null +++ b/packages/openapi/objectGenerator.ts @@ -0,0 +1,291 @@ +import type { OpenApiSchemaObject, OpenApiSchemaOrReference, OpenApiSpec } from './types.js' + +const SCHEMA_REF_PREFIX = '#/components/schemas/' +const MAX_DEPTH = 1 + +type GenContext = { + spec: OpenApiSpec + timestamp: number +} + +const ID_PREFIXES: Record = { + account: 'acct', + apple_pay_domain: 'apftw', + application_fee: 'fee', + balance_transaction: 'txn', + 'billing.alert': 'alrt', + 'billing.credit_balance_transaction': 'cbtxn', + 'billing.credit_grant': 'credgr', + 'billing.meter': 'mtr', + 'billing_portal.configuration': 'bpc', + charge: 'ch', + 'checkout.session': 'cs', + 'climate.order': 'climorder', + 'climate.product': 'climsku', + 'climate.supplier': 'climsup', + country_spec: 'cspec', + coupon: 'cpn', + credit_note: 'cn', + customer: 'cus', + dispute: 'dp', + event: 'evt', + exchange_rate: 'xr', + file_link: 'link', + file: 'file', + invoiceitem: 'ii', + invoice: 'in', + payment_intent: 'pi', + payment_link: 'plink', + payment_method: 'pm', + payout: 'po', + plan: 'plan', + price: 'price', + product: 'prod', + promotion_code: 'promo', + quote: 'qt', + refund: 're', + setup_intent: 'seti', + subscription: 'sub', + subscription_schedule: 'sub_sched', + tax_id: 'txi', + tax_rate: 'txr', + topup: 'tu', + transfer: 'tr', + webhook_endpoint: 'we', + v2_core_account: 'acct', + 'v2.core.event_destination': 'ed', + 'v2.core.event': 'evt', +} + +export type GenerateObjectsOptions = { + tableName?: string + /** Unix timestamp (seconds) used for `created`, `updated`, and `*_at` integer fields. Defaults to now. */ + createdTimestamp?: number +} + +/** + * Generate `count` objects conforming to a named schema in the OpenAPI spec. + * Resolves `$ref`, `oneOf`/`anyOf`/`allOf`, enums, and nested structures, + * filling each field with a type-appropriate value. + */ +export function generateObjectsFromSchema( + spec: OpenApiSpec, + schemaName: string, + count: number, + options?: GenerateObjectsOptions +): Record[] { + const schemaOrRef = spec.components?.schemas?.[schemaName] + if (!schemaOrRef) { + throw new Error(`Schema "${schemaName}" not found in spec`) + } + + const schema = resolveRef(schemaOrRef, spec) + const resourceId = schema['x-resourceId'] ?? schemaName + const prefix = resolveIdPrefix(resourceId, options?.tableName) + const ctx: GenContext = { + spec, + timestamp: options?.createdTimestamp ?? Math.floor(Date.now() / 1000), + } + + const template = generateObject(schema, ctx, 0) as Record + if (schema.properties?.object) { + template.object = resourceId + } + + const objects: Record[] = [] + for (let i = 0; i < count; i++) { + objects.push({ ...template, id: `${prefix}_${i.toString(36).padStart(12, '0')}` }) + } + + return objects +} + +/** + * Find the component schema name for a given `x-resourceId`. + * Returns `undefined` if no matching schema exists. + */ +export function findSchemaNameByResourceId( + spec: OpenApiSpec, + resourceId: string +): string | undefined { + const schemas = spec.components?.schemas + if (!schemas) return undefined + + for (const [name, schemaOrRef] of Object.entries(schemas)) { + if ('$ref' in schemaOrRef) continue + if (schemaOrRef['x-resourceId'] === resourceId) return name + } + return undefined +} + +function resolveIdPrefix(resourceId: string, tableName?: string): string { + if (tableName) { + const byTable = ID_PREFIXES[tableName] + if (byTable) return byTable + } + return ID_PREFIXES[resourceId] ?? resourceId.replace(/\./g, '_').slice(0, 6) +} + +function resolveRef( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec, + seen = new Set() +): OpenApiSchemaObject { + if (!('$ref' in schemaOrRef)) return schemaOrRef + const ref = schemaOrRef.$ref + if (!ref.startsWith(SCHEMA_REF_PREFIX) || seen.has(ref)) return {} + seen.add(ref) + const name = ref.slice(SCHEMA_REF_PREFIX.length) + const resolved = spec.components?.schemas?.[name] + if (!resolved) return {} + return resolveRef(resolved, spec, seen) +} + +function generateObject(schema: OpenApiSchemaObject, ctx: GenContext, depth: number): unknown { + const merged = mergeComposed(schema, ctx.spec, depth) + const properties = merged.properties ?? schema.properties + if (!properties) return {} + + const obj: Record = {} + for (const [key, propRef] of Object.entries(properties)) { + obj[key] = generateValue(propRef, ctx, key, depth) + } + return obj +} + +function mergeComposed( + schema: OpenApiSchemaObject, + spec: OpenApiSpec, + depth: number +): OpenApiSchemaObject { + const composites = schema.allOf ?? schema.anyOf + if (!composites?.length) return schema + + const mergedProps: Record = { + ...(schema.properties ?? {}), + } + for (const sub of composites) { + const resolved = resolveRef(sub, spec) + if (resolved.properties) { + Object.assign(mergedProps, resolved.properties) + } + if (resolved.allOf || resolved.anyOf) { + const nested = mergeComposed(resolved, spec, depth) + if (nested.properties) { + Object.assign(mergedProps, nested.properties) + } + } + } + return { ...schema, properties: mergedProps } +} + +function generateValue( + schemaOrRef: OpenApiSchemaOrReference, + ctx: GenContext, + fieldName: string, + depth: number +): unknown { + const schema = resolveRef(schemaOrRef, ctx.spec) + + if (schema.nullable) { + return null + } + + if (schema.oneOf?.length) { + return pickFromOneOf(schema.oneOf, ctx, fieldName, depth) + } + if (schema.anyOf?.length) { + return pickFromOneOf(schema.anyOf, ctx, fieldName, depth) + } + if (schema.allOf?.length) { + const merged = mergeComposed(schema, ctx.spec, depth) + return generateObject(merged, ctx, depth) + } + + if (schema.enum?.length) { + return schema.enum[0] + } + + switch (schema.type) { + case 'string': + return generateString(schema, fieldName) + case 'integer': + return generateInteger(fieldName, ctx.timestamp) + case 'number': + return 0.0 + case 'boolean': + return false + case 'array': + return generateArray(schema, ctx, fieldName, depth) + case 'object': + if (depth >= MAX_DEPTH) return {} + return generateObject(schema, ctx, depth + 1) + } + + if (schema.properties) { + if (depth >= MAX_DEPTH) return {} + return generateObject(schema, ctx, depth + 1) + } + + return null +} + +function pickFromOneOf( + variants: OpenApiSchemaOrReference[], + ctx: GenContext, + fieldName: string, + depth: number +): unknown { + for (const variant of variants) { + const resolved = resolveRef(variant, ctx.spec) + if (resolved.type === 'string') return generateString(resolved, fieldName) + if (resolved.type === 'integer') return generateInteger(fieldName, ctx.timestamp) + if (resolved.type === 'number') return 0.0 + if (resolved.type === 'boolean') return false + if (resolved.enum?.length) return resolved.enum[0] + } + if (depth >= MAX_DEPTH) return null + const first = resolveRef(variants[0], ctx.spec) + if (first.properties || first.allOf || first.anyOf) { + return generateObject(first, ctx, depth + 1) + } + return null +} + +function generateString(schema: OpenApiSchemaObject, fieldName: string): string { + if (schema.enum?.length) return String(schema.enum[0]) + if (schema.format === 'date-time') return new Date().toISOString() + if (schema.format === 'uri' || schema.format === 'url') return 'https://example.com' + if (fieldName === 'currency') return 'usd' + if (fieldName === 'email') return 'test@example.com' + if (fieldName === 'phone') return '+15555555555' + if (fieldName === 'url' || fieldName.endsWith('_url')) return 'https://example.com' + return `test_${fieldName}` +} + +function generateInteger(fieldName: string, timestamp: number): number { + if (fieldName === 'created' || fieldName === 'updated') { + return timestamp + } + if (fieldName.endsWith('_at')) { + return timestamp + } + return 0 +} + +function generateArray( + schema: OpenApiSchemaObject, + ctx: GenContext, + _fieldName: string, + depth: number +): unknown[] { + if (!schema.items || depth >= MAX_DEPTH) return [] + const itemSchema = resolveRef(schema.items, ctx.spec) + if (itemSchema.properties || itemSchema.allOf || itemSchema.anyOf) { + if (depth + 1 >= MAX_DEPTH) return [] + return [generateObject(itemSchema, ctx, depth + 1)] + } + if (itemSchema.type === 'string') return ['test_item'] + if (itemSchema.type === 'integer') return [0] + return [] +} diff --git a/packages/source-stripe/src/client.test.ts b/packages/source-stripe/src/client.test.ts index 8d555f380..aa6e45a55 100644 --- a/packages/source-stripe/src/client.test.ts +++ b/packages/source-stripe/src/client.test.ts @@ -1,7 +1,14 @@ -import { describe, expect, it } from 'vitest' +import { afterEach, describe, expect, it, vi } from 'vitest' import { makeClient, StripeRequestError } from './client.js' import { getProxyUrl } from './transport.js' +const originalFetch = globalThis.fetch + +afterEach(() => { + vi.useRealTimers() + globalThis.fetch = originalFetch +}) + describe('getProxyUrl', () => { it('prefers HTTPS_PROXY over HTTP_PROXY', () => { expect( @@ -44,4 +51,51 @@ describe('makeClient', () => { expect(err.requestId).toBe('req_123') expect(err.message).toBe('Invalid API Key') }) + + it('retries transient GET failures and eventually succeeds', async () => { + vi.useFakeTimers() + const fetchMock = vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + error: { type: 'api_error', message: 'Temporary outage' }, + }), + { status: 500 } + ) + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ id: 'acct_test', object: 'account', created: 123 }), { + status: 200, + }) + ) + globalThis.fetch = fetchMock + + const client = makeClient({ api_key: 'sk_test_fake', base_url: 'http://stripe.test' }, {}) + const pending = client.getAccount() + await vi.runAllTimersAsync() + + await expect(pending).resolves.toMatchObject({ id: 'acct_test', created: 123 }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('does not retry auth failures on GET requests', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce( + new Response( + JSON.stringify({ + error: { + type: 'invalid_request_error', + message: 'Invalid API Key provided: sk_test_bad', + }, + }), + { status: 401 } + ) + ) + globalThis.fetch = fetchMock + + const client = makeClient({ api_key: 'sk_test_bad', base_url: 'http://stripe.test' }, {}) + + await expect(client.getAccount()).rejects.toThrow('Invalid API Key provided: sk_test_bad') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/source-stripe/src/client.ts b/packages/source-stripe/src/client.ts index cdc497f3c..26707c153 100644 --- a/packages/source-stripe/src/client.ts +++ b/packages/source-stripe/src/client.ts @@ -7,6 +7,7 @@ import { type StripeApiList, type StripeWebhookEndpoint, } from '@stripe/sync-openapi' +import { withHttpRetry } from './retry.js' import { stripeEventSchema, type StripeEvent } from './spec.js' import { fetchWithProxy, parsePositiveInteger, type TransportEnv } from './transport.js' @@ -109,9 +110,20 @@ export function makeClient(config: StripeClientConfig, env: TransportEnv = proce return json } + async function requestWithRetry( + method: string, + path: string, + params?: Record + ): Promise { + if (method === 'GET') { + return withHttpRetry(() => request(method, path, params)) + } + return request(method, path, params) + } + return { async getAccount(): Promise { - const json = await request('GET', '/v1/account') + const json = await requestWithRetry('GET', '/v1/account') return StripeAccountSchema.parse(json) }, @@ -124,14 +136,14 @@ export function makeClient(config: StripeClientConfig, env: TransportEnv = proce if (params.limit) query.limit = params.limit if (params.starting_after) query.starting_after = params.starting_after if (params.created?.gt) query['created[gt]'] = params.created.gt - const json = await request('GET', '/v1/events', query) + const json = await requestWithRetry('GET', '/v1/events', query) return StripeApiListSchema(stripeEventSchema).parse(json) }, async listWebhookEndpoints(params?: { limit?: number }): Promise> { - const json = await request('GET', '/v1/webhook_endpoints', params) + const json = await requestWithRetry('GET', '/v1/webhook_endpoints', params) return StripeApiListSchema(StripeWebhookEndpointSchema).parse(json) }, @@ -140,12 +152,12 @@ export function makeClient(config: StripeClientConfig, env: TransportEnv = proce enabled_events: string[] metadata?: Record }): Promise { - const json = await request('POST', '/v1/webhook_endpoints', params) + const json = await requestWithRetry('POST', '/v1/webhook_endpoints', params) return StripeWebhookEndpointSchema.parse(json) }, async deleteWebhookEndpoint(id: string): Promise { - await request('DELETE', `/v1/webhook_endpoints/${encodeURIComponent(id)}`) + await requestWithRetry('DELETE', `/v1/webhook_endpoints/${encodeURIComponent(id)}`) }, } } diff --git a/packages/source-stripe/src/index.test.ts b/packages/source-stripe/src/index.test.ts index c1faecf9d..905fc8ca3 100644 --- a/packages/source-stripe/src/index.test.ts +++ b/packages/source-stripe/src/index.test.ts @@ -2,7 +2,7 @@ import fs from 'node:fs' import path from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { StripeEvent } from './spec.js' -import type { StripeClient } from './client.js' +import { StripeRequestError, type StripeClient } from './client.js' import type { ConfiguredCatalog, Message, @@ -610,6 +610,163 @@ describe('StripeSource', () => { expect(traceError.message).toContain('Connection refused') }) + it('emits TraceMessage error when getAccount fails before parallel backfill pagination', async () => { + const listFn = vi.fn() + + const registry: Record = { + customers: makeConfig({ + order: 1, + tableName: 'customers', + supportsCreatedFilter: true, + listFn: listFn as ResourceConfig['listFn'], + }), + } + + const mockClient = { + getAccount: vi.fn().mockRejectedValueOnce( + new StripeRequestError(401, { + type: 'invalid_request_error', + message: 'Invalid API Key provided: sk_test_bad', + }) + ), + } as unknown as StripeClient + + const messages = await collect( + listApiBackfill({ + catalog: catalog({ name: 'customers' }), + state: undefined, + registry, + client: mockClient, + rateLimiter: async () => 0, + backfillConcurrency: 2, + }) + ) + + expect(messages).toHaveLength(2) + expect(listFn).not.toHaveBeenCalled() + expect(messages[0]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'customers', status: 'started' }, + }, + }) + + const errorMsg = messages[1] as TraceMessage + expect(errorMsg.trace.trace_type).toBe('error') + const traceError = ( + errorMsg.trace as { + trace_type: 'error' + error: { failure_type: string; message: string; stream?: string } + } + ).error + expect(traceError.failure_type).toBe('system_error') + expect(traceError.message).toContain('Invalid API Key') + expect(traceError.stream).toBe('customers') + }) + + it('emits TraceMessage error for Invalid API Key on sequential streams', async () => { + const listFn = vi.fn().mockRejectedValueOnce( + new StripeRequestError(401, { + type: 'invalid_request_error', + message: 'Invalid API Key provided: sk_test_bad', + }) + ) + + const registry: Record = { + tax_ids: makeConfig({ + order: 1, + tableName: 'tax_ids', + supportsCreatedFilter: false, + listFn: listFn as ResourceConfig['listFn'], + }), + } + + vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) + const messages = await collect( + source.read({ config, catalog: catalog({ name: 'tax_ids', primary_key: [['id']] }) }) + ) + + expect(messages).toHaveLength(2) + const errorMsg = messages[1] as TraceMessage + expect(errorMsg.trace.trace_type).toBe('error') + const traceError = ( + errorMsg.trace as { + trace_type: 'error' + error: { failure_type: string; message: string; stream?: string } + } + ).error + expect(traceError.failure_type).toBe('system_error') + expect(traceError.message).toContain('Invalid API Key') + expect(traceError.stream).toBe('tax_ids') + }) + + it('does not treat near-miss auth errors as skippable', async () => { + const listFn = vi + .fn() + .mockRejectedValueOnce(new Error('Authentication failed: must provide a valid API key')) + + const registry: Record = { + customers: makeConfig({ + order: 1, + tableName: 'customers', + listFn: listFn as ResourceConfig['listFn'], + }), + } + + vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) + const messages = await collect( + source.read({ config, catalog: catalog({ name: 'customers', primary_key: [['id']] }) }) + ) + + expect(messages).toHaveLength(2) + expect(messages[1]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'error', + error: { + failure_type: 'system_error', + stream: 'customers', + }, + }, + }) + }) + + it('marks known skippable Stripe list errors as complete without emitting error traces', async () => { + const listFn = vi + .fn() + .mockRejectedValueOnce(new Error('This object is only available in testmode')) + + const registry: Record = { + invoices: makeConfig({ + order: 1, + tableName: 'invoices', + listFn: listFn as ResourceConfig['listFn'], + }), + } + + vi.mocked(buildResourceRegistry).mockReturnValue(registry as any) + const messages = await collect( + source.read({ config, catalog: catalog({ name: 'invoices', primary_key: [['id']] }) }) + ) + + expect(messages).toHaveLength(2) + expect(messages[0]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'invoices', status: 'started' }, + }, + }) + expect(messages[1]).toMatchObject({ + type: 'trace', + trace: { + trace_type: 'stream_status', + stream_status: { stream: 'invoices', status: 'complete' }, + }, + }) + }) + it('continues to next stream after error on previous stream', async () => { const failingListFn = vi.fn().mockRejectedValueOnce(new Error('Connection refused')) const successListFn = vi.fn().mockResolvedValueOnce({ @@ -1828,6 +1985,46 @@ describe('StripeSource', () => { } }) + it('does not assume cursor pagination when the endpoint does not support it', async () => { + const listFn = vi.fn().mockResolvedValue({ + data: [{ id: 'report_type_1', name: 'One shot' }], + has_more: true, + }) + + const registry: Record = { + reporting_report_types: makeConfig({ + order: 1, + tableName: 'reporting_report_types', + supportsCreatedFilter: false, + supportsLimit: false, + supportsForwardPagination: false, + listFn: listFn as ResourceConfig['listFn'], + }), + } + + const mockClient = {} as unknown as StripeClient + const rateLimiter: RateLimiter = async () => 0 + + const messages = await collect( + listApiBackfill({ + catalog: catalog({ name: 'reporting_report_types' }), + state: undefined, + registry, + client: mockClient, + rateLimiter, + }) + ) + + expect(listFn).toHaveBeenCalledTimes(1) + expect(listFn).toHaveBeenCalledWith({}) + + const states = messages.filter((m): m is SourceStateMessage => m.type === 'source_state') + expect(states.at(-1)?.source_state.data).toMatchObject({ + status: 'complete', + page_cursor: null, + }) + }) + it('parallel and sequential streams coexist in the same catalog', async () => { const parallelListFn = vi.fn().mockResolvedValue({ data: [{ id: 'cus_1' }], diff --git a/packages/source-stripe/src/index.ts b/packages/source-stripe/src/index.ts index 85470cfda..04f5479d1 100644 --- a/packages/source-stripe/src/index.ts +++ b/packages/source-stripe/src/index.ts @@ -335,6 +335,7 @@ export default createStripeSource() // MARK: - Re-exports +export { expandState } from './src-list-api.js' export { buildResourceRegistry, DEFAULT_SYNC_OBJECTS } from './resourceRegistry.js' export { catalogFromRegistry } from './catalog.js' export { SpecParser, OPENAPI_RESOURCE_TABLE_ALIASES } from './openapi/specParser.js' diff --git a/packages/source-stripe/src/resourceRegistry.test.ts b/packages/source-stripe/src/resourceRegistry.test.ts new file mode 100644 index 000000000..39e12731e --- /dev/null +++ b/packages/source-stripe/src/resourceRegistry.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it } from 'vitest' +import type { OpenApiSpec } from '@stripe/sync-openapi' +import { buildResourceRegistry } from './resourceRegistry.js' + +const v2CreatedSpec: OpenApiSpec = { + openapi: '3.0.0', + paths: { + '/v2/core/accounts': { + get: { + parameters: [{ name: 'limit', in: 'query' }], + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + data: { + type: 'array', + items: { $ref: '#/components/schemas/v2.core.account' }, + }, + next_page_url: { type: 'string', nullable: true }, + previous_page_url: { type: 'string', nullable: true }, + }, + }, + }, + }, + }, + }, + }, + }, + '/v2/core/events': { + get: { + parameters: [ + { + name: 'created', + in: 'query', + schema: { + type: 'object', + properties: { + gte: { type: 'string', format: 'date-time' }, + lt: { type: 'string', format: 'date-time' }, + }, + }, + }, + { name: 'limit', in: 'query' }, + ], + responses: { + '200': { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + data: { + type: 'array', + items: { $ref: '#/components/schemas/v2.core.event' }, + }, + next_page_url: { type: 'string', nullable: true }, + previous_page_url: { type: 'string', nullable: true }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + components: { + schemas: { + 'v2.core.account': { + 'x-resourceId': 'v2.core.account', + type: 'object', + properties: { + id: { type: 'string' }, + }, + }, + 'v2.core.event': { + 'x-resourceId': 'v2.core.event', + type: 'object', + properties: { + id: { type: 'string' }, + created: { type: 'string', format: 'date-time' }, + }, + }, + }, + }, +} + +describe('buildResourceRegistry', () => { + it('keeps v2 created filter support when the spec advertises it', () => { + const registry = buildResourceRegistry(v2CreatedSpec, 'sk_test_fake') + + expect(registry.v2_core_accounts?.supportsCreatedFilter).toBe(false) + expect(registry.v2_core_events?.supportsCreatedFilter).toBe(true) + }) +}) diff --git a/packages/source-stripe/src/resourceRegistry.ts b/packages/source-stripe/src/resourceRegistry.ts index 20bb81613..ebbb5a7dc 100644 --- a/packages/source-stripe/src/resourceRegistry.ts +++ b/packages/source-stripe/src/resourceRegistry.ts @@ -1,19 +1,53 @@ import type { ResourceConfig } from './types.js' -import type { OpenApiSpec, NestedEndpoint } from '@stripe/sync-openapi' +import type { ListFn, ListParams, OpenApiSpec, NestedEndpoint } from '@stripe/sync-openapi' import { discoverListEndpoints, discoverNestedEndpoints, - isV2Path, buildListFn, buildRetrieveFn, + isV2Path, resolveTableName, OPENAPI_RESOURCE_TABLE_ALIASES, } from '@stripe/sync-openapi' import { fetchWithProxy } from './transport.js' +import { withHttpRetry } from './retry.js' const apiFetch: typeof globalThis.fetch = (input, init) => fetchWithProxy(input as URL | string, init ?? {}) +/** + * Wrap a raw list function so only params the endpoint actually supports are forwarded. + * Some Stripe endpoints (e.g. reporting_report_types) don't accept `limit` or + * `starting_after`; sending them causes 400 errors. + */ +function buildSpecAwareListFn( + baseListFn: ListFn, + options: { + isV2: boolean + supportsLimit: boolean + supportsStartingAfter: boolean + supportsEndingBefore: boolean + supportsCreatedFilter: boolean + } +): ListFn { + return (params: ListParams) => { + const next: ListParams = {} + if (options.supportsLimit && params.limit != null) { + next.limit = params.limit + } + if ((options.isV2 || options.supportsStartingAfter) && params.starting_after) { + next.starting_after = params.starting_after + } + if (options.supportsEndingBefore && params.ending_before) { + next.ending_before = params.ending_before + } + if (options.supportsCreatedFilter && params.created) { + next.created = params.created + } + return baseListFn(next) + } +} + /** * The default set of table names synced when no explicit selection is made. * These correspond to the resources that were previously hardcoded with sync: true. @@ -63,8 +97,7 @@ export function buildResourceRegistry( const seenNested = new Set() for (const [tableName, endpoint] of endpoints) { - const v2 = isV2Path(endpoint.apiPath) - + const isV2 = isV2Path(endpoint.apiPath) const children = nestedEndpoints .filter((n: NestedEndpoint) => n.parentTableName === tableName) .map((n: NestedEndpoint) => ({ @@ -75,15 +108,28 @@ export function buildResourceRegistry( supportsPagination: n.supportsPagination, })) + const rawListFn = buildListFn(apiKey, endpoint.apiPath, apiFetch, apiVersion, baseUrl) + const rawRetrieveFn = buildRetrieveFn(apiKey, endpoint.apiPath, apiFetch, apiVersion, baseUrl) + const config: ResourceConfig = { order: 1, tableName, - supportsCreatedFilter: !v2 && endpoint.supportsCreatedFilter, + supportsCreatedFilter: endpoint.supportsCreatedFilter, supportsLimit: endpoint.supportsLimit, + supportsForwardPagination: isV2 || endpoint.supportsStartingAfter, sync: true, dependencies: [], - listFn: buildListFn(apiKey, endpoint.apiPath, apiFetch, apiVersion, baseUrl), - retrieveFn: buildRetrieveFn(apiKey, endpoint.apiPath, apiFetch, apiVersion, baseUrl), + listFn: buildSpecAwareListFn( + (params) => withHttpRetry(() => rawListFn(params)), + { + isV2, + supportsLimit: endpoint.supportsLimit, + supportsStartingAfter: endpoint.supportsStartingAfter, + supportsEndingBefore: endpoint.supportsEndingBefore, + supportsCreatedFilter: endpoint.supportsCreatedFilter, + } + ), + retrieveFn: (id) => withHttpRetry(() => rawRetrieveFn(id)), nestedResources: children.length > 0 ? children : undefined, } registry[tableName] = config @@ -103,6 +149,7 @@ export function buildResourceRegistry( tableName: nested.tableName, supportsCreatedFilter: false, supportsLimit: nested.supportsPagination, + supportsForwardPagination: nested.supportsPagination, sync: false, dependencies: [], listFn: undefined, diff --git a/packages/source-stripe/src/retry.ts b/packages/source-stripe/src/retry.ts new file mode 100644 index 000000000..3cd65341c --- /dev/null +++ b/packages/source-stripe/src/retry.ts @@ -0,0 +1,113 @@ +const BACKOFF_BASE_MS = 1000 +const BACKOFF_MAX_MS = 32000 +const MAX_RETRIES = 5 + +const RETRYABLE_NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ETIMEDOUT', + 'EAI_AGAIN', + 'ENOTFOUND', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_SOCKET', +]) + +export type HttpRetryOptions = { + maxRetries?: number + baseDelayMs?: number + maxDelayMs?: number +} + +export function getHttpErrorStatus(err: unknown): number | undefined { + if (!err || typeof err !== 'object') return undefined + + if ('status' in err && typeof err.status === 'number') { + return err.status + } + + if ('statusCode' in err && typeof err.statusCode === 'number') { + return err.statusCode + } + + if ('code' in err && typeof err.code === 'number') { + return err.code + } + + return undefined +} + +function getNestedErrorCode(err: unknown): string | undefined { + if (!err || typeof err !== 'object') return undefined + + if ('code' in err && typeof err.code === 'string') { + return err.code + } + + if ('cause' in err) { + return getNestedErrorCode(err.cause) + } + + return undefined +} + +export function isRetryableHttpError(err: unknown): boolean { + const status = getHttpErrorStatus(err) + if (status === 429 || (status !== undefined && status >= 500)) { + return true + } + if (status !== undefined) { + return false + } + + if (!(err instanceof Error)) { + return false + } + + if (err.name === 'AbortError' || err.name === 'TimeoutError') { + return true + } + + const code = getNestedErrorCode(err) + if (code && RETRYABLE_NETWORK_CODES.has(code)) { + return true + } + + const message = err.message.toLowerCase() + return ( + message.includes('fetch failed') || message.includes('network') || message.includes('timeout') + ) +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +export async function withHttpRetry( + fn: () => Promise, + opts: HttpRetryOptions = {} +): Promise { + const maxRetries = opts.maxRetries ?? MAX_RETRIES + const maxDelayMs = opts.maxDelayMs ?? BACKOFF_MAX_MS + let delayMs = opts.baseDelayMs ?? BACKOFF_BASE_MS + + for (let attempt = 0; ; attempt++) { + try { + return await fn() + } catch (err) { + if (attempt >= maxRetries || !isRetryableHttpError(err)) { + throw err + } + + const status = getHttpErrorStatus(err) + const errName = err instanceof Error ? err.name : 'UnknownError' + console.error( + `[source-stripe] retry attempt=${attempt + 1}/${maxRetries} delay=${delayMs}ms status=${status ?? 'n/a'} error=${errName}` + ) + + await sleep(delayMs) + delayMs = Math.min(delayMs * 2, maxDelayMs) + } + } +} diff --git a/packages/source-stripe/src/src-list-api.ts b/packages/source-stripe/src/src-list-api.ts index 0696eb5ce..dcc6c7646 100644 --- a/packages/source-stripe/src/src-list-api.ts +++ b/packages/source-stripe/src/src-list-api.ts @@ -222,6 +222,7 @@ async function* paginateSegment(opts: { numSegments: number streamName: string supportsLimit: boolean + supportsForwardPagination: boolean backfillLimit?: number totalEmitted: { count: number } rateLimiter: RateLimiter @@ -234,6 +235,7 @@ async function* paginateSegment(opts: { numSegments, streamName, supportsLimit, + supportsForwardPagination, backfillLimit, totalEmitted, rateLimiter, @@ -246,10 +248,10 @@ async function* paginateSegment(opts: { const params: Record = { created: { gte: segment.gte, lt: segment.lt }, } - if (supportsLimit !== false) { + if (supportsForwardPagination && supportsLimit !== false) { params.limit = 100 } - if (pageCursor) { + if (supportsForwardPagination && pageCursor) { params.starting_after = pageCursor } @@ -262,7 +264,7 @@ async function* paginateSegment(opts: { totalEmitted.count++ } - hasMore = response.has_more + hasMore = supportsForwardPagination && response.has_more if (response.pageCursor) { pageCursor = response.pageCursor } else if (response.data.length > 0) { @@ -308,10 +310,14 @@ async function* sequentialBackfillStream(opts: { if (drainQueue) yield* drainQueue() const params: Record = {} - if (resourceConfig.supportsLimit !== false) { + // `!== false` treats undefined as "supports pagination" for backward compat. + if ( + resourceConfig.supportsForwardPagination !== false && + resourceConfig.supportsLimit !== false + ) { params.limit = 100 } - if (pageCursor) { + if (resourceConfig.supportsForwardPagination !== false && pageCursor) { params.starting_after = pageCursor } @@ -326,7 +332,7 @@ async function* sequentialBackfillStream(opts: { totalEmitted++ } - hasMore = response.has_more + hasMore = resourceConfig.supportsForwardPagination !== false && response.has_more if (response.pageCursor) { pageCursor = response.pageCursor } else if (response.data.length > 0) { @@ -455,6 +461,7 @@ export async function* listApiBackfill(opts: { numSegments, streamName: stream.name, supportsLimit: resourceConfig.supportsLimit !== false, + supportsForwardPagination: resourceConfig.supportsForwardPagination !== false, backfillLimit: streamBackfillLimit, totalEmitted, rateLimiter, diff --git a/packages/source-stripe/src/types.ts b/packages/source-stripe/src/types.ts index cb0f70233..678b1be7c 100644 --- a/packages/source-stripe/src/types.ts +++ b/packages/source-stripe/src/types.ts @@ -34,6 +34,8 @@ export type ResourceConfig = BaseResourceConfig & { retrieveFn?: RetrieveFn /** Whether the list API supports the `limit` parameter */ supportsLimit?: boolean + /** Whether the list API supports forward cursor pagination for repeated page fetches. */ + supportsForwardPagination?: boolean /** Nested child resources discovered from the spec (e.g. subscription items under subscriptions) */ nestedResources?: { tableName: string diff --git a/packages/test-utils/README.md b/packages/test-utils/README.md new file mode 100644 index 000000000..094a562a7 --- /dev/null +++ b/packages/test-utils/README.md @@ -0,0 +1,21 @@ +# @stripe/sync-test-utils + +Test utilities for the sync-engine integration tests: + +- **Hono HTTP server** that discovers every listable Stripe endpoint from the OpenAPI spec and serves Stripe-compatible list/retrieve responses backed by Postgres +- **Docker Postgres 18 helper** — spins up a disposable container with SSL, waits for readiness, and cleans up on exit +- **DB seeding** — generates OpenAPI-schema-compliant objects (`generateObjectsFromSchema` from `@stripe/sync-openapi`) and bulk-inserts them into Postgres with configurable `created` timestamp ranges + +## Quick start + +```sh +pnpm --filter @stripe/sync-test-utils build +pnpm --filter @stripe/sync-test-utils exec sync-test-utils-server +``` + +## Notes + +- No external mock server is required. Objects are generated from OpenAPI schemas and stored directly in Postgres. +- If `POSTGRES_URL` is not provided, the server starts an internal `postgres:18` Docker container automatically. +- List query parameters are validated against each endpoint's OpenAPI parameter definitions, including v2 endpoints. +- Seeding supports spreading `created` timestamps across a range via `applyCreatedTimestampRange`. diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json new file mode 100644 index 000000000..938814600 --- /dev/null +++ b/packages/test-utils/package.json @@ -0,0 +1,35 @@ +{ + "name": "@stripe/sync-test-utils", + "version": "0.1.0", + "private": true, + "type": "module", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "bin": { + "sync-test-utils-server": "./dist/bin/start-test-server.js" + }, + "scripts": { + "build": "tsc", + "test": "vitest" + }, + "files": [ + "dist", + "src" + ], + "dependencies": { + "@hono/node-server": "^1.19.11", + "@stripe/sync-destination-postgres": "workspace:*", + "@stripe/sync-openapi": "workspace:*", + "hono": "^4.12.8", + "pg": "^8.16.3" + }, + "devDependencies": { + "@types/node": "^24.10.1", + "@types/pg": "^8.15.5", + "vitest": "^3.2.4" + } +} diff --git a/packages/test-utils/src/__tests__/filters.test.ts b/packages/test-utils/src/__tests__/filters.test.ts new file mode 100644 index 000000000..bb6db10d8 --- /dev/null +++ b/packages/test-utils/src/__tests__/filters.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from 'vitest' +import { validateQueryAgainstOpenApi } from '../openapi/filters.js' +import type { EndpointQueryParam } from '../openapi/endpoints.js' + +describe('validateQueryAgainstOpenApi', () => { + it('accepts object-style nested filters and scalar filters', () => { + const params: EndpointQueryParam[] = [ + { + name: 'created', + required: false, + schema: { + type: 'object', + properties: { + gt: { type: 'integer' }, + lte: { type: 'integer' }, + }, + }, + }, + { name: 'limit', required: false, schema: { type: 'integer' } }, + ] + + const validated = validateQueryAgainstOpenApi( + new URLSearchParams([ + ['created[gt]', '1000'], + ['limit', '10'], + ]), + params + ) + expect(validated.ok).toBe(true) + if (!validated.ok) return + expect(validated.forward.get('created[gt]')).toBe('1000') + expect(validated.forward.get('limit')).toBe('10') + }) + + it('unwraps anyOf to find object properties', () => { + const params: EndpointQueryParam[] = [ + { + name: 'created', + required: false, + schema: { + anyOf: [ + { type: 'string' }, + { + type: 'object', + properties: { + gt: { type: 'integer' }, + gte: { type: 'integer' }, + lt: { type: 'integer' }, + lte: { type: 'integer' }, + }, + }, + ], + } as any, + }, + { name: 'limit', required: false, schema: { type: 'integer' } }, + ] + + const validated = validateQueryAgainstOpenApi( + new URLSearchParams([ + ['created[gte]', '1000'], + ['created[lt]', '2000'], + ['limit', '10'], + ]), + params + ) + expect(validated.ok).toBe(true) + if (!validated.ok) return + expect(validated.forward.get('created[gte]')).toBe('1000') + expect(validated.forward.get('created[lt]')).toBe('2000') + }) + + it('rejects unknown filters with allowed list', () => { + const params: EndpointQueryParam[] = [ + { name: 'limit', required: false, schema: { type: 'integer' } }, + ] + + const validated = validateQueryAgainstOpenApi(new URLSearchParams([['foo', 'bar']]), params) + expect(validated.ok).toBe(false) + if (validated.ok) return + expect(validated.statusCode).toBe(400) + expect(validated.allowed).toEqual(['limit']) + }) +}) diff --git a/packages/test-utils/src/__tests__/storage.test.ts b/packages/test-utils/src/__tests__/storage.test.ts new file mode 100644 index 000000000..143db4181 --- /dev/null +++ b/packages/test-utils/src/__tests__/storage.test.ts @@ -0,0 +1,21 @@ +import type pg from 'pg' +import { describe, expect, it, vi } from 'vitest' +import { ensureObjectTable } from '../db/storage.js' + +describe('ensureObjectTable', () => { + it('creates the pagination index used by the fake Stripe server', async () => { + const query = vi.fn().mockResolvedValue({ rows: [] }) + const pool = { query } as unknown as pg.Pool + + await ensureObjectTable(pool, 'stripe', 'customers') + + expect(query).toHaveBeenCalledTimes(2) + expect(normalizeSql(query.mock.calls[1]?.[0] as string)).toBe( + 'CREATE INDEX IF NOT EXISTS "customers_created_id_idx" ON "stripe"."customers" ("created" DESC, "id" DESC)' + ) + }) +}) + +function normalizeSql(sql: string): string { + return sql.replace(/\s+/g, ' ').trim() +} diff --git a/packages/test-utils/src/bin/start-test-server.ts b/packages/test-utils/src/bin/start-test-server.ts new file mode 100644 index 000000000..8cb2bf470 --- /dev/null +++ b/packages/test-utils/src/bin/start-test-server.ts @@ -0,0 +1,51 @@ +#!/usr/bin/env node +import { createStripeListServer } from '../server/createStripeListServer.js' + +async function main(): Promise { + const argv = parseArgs(process.argv.slice(2)) + const server = await createStripeListServer({ + port: argv.port ? Number(argv.port) : undefined, + host: argv.host, + postgresUrl: argv['postgres-url'], + apiVersion: argv['api-version'], + openApiSpecPath: argv['openapi-spec-path'], + schema: argv.schema, + accountCreated: argv['account-created'] ? Number(argv['account-created']) : undefined, + }) + + process.stderr.write( + `sync-test-utils server listening at ${server.url} (postgres_mode=${server.postgresMode})\n` + ) + + await new Promise((resolve) => { + const stop = async () => { + await server.close().catch(() => undefined) + resolve() + } + process.once('SIGINT', () => void stop()) + process.once('SIGTERM', () => void stop()) + }) +} + +function parseArgs(args: string[]): Record { + const parsed: Record = {} + for (let i = 0; i < args.length; i++) { + const token = args[i] + if (!token.startsWith('--')) continue + const key = token.slice(2) + const next = args[i + 1] + if (!next || next.startsWith('--')) { + parsed[key] = 'true' + continue + } + parsed[key] = next + i += 1 + } + return parsed +} + +main().catch((error) => { + const message = error instanceof Error ? error.message : String(error) + process.stderr.write(`sync-test-utils server failed: ${message}\n`) + process.exit(1) +}) diff --git a/packages/test-utils/src/db/storage.ts b/packages/test-utils/src/db/storage.ts new file mode 100644 index 000000000..ef9c16c78 --- /dev/null +++ b/packages/test-utils/src/db/storage.ts @@ -0,0 +1,110 @@ +import type pg from 'pg' +import { buildCreateTableWithSchema, runSqlAdditive } from '@stripe/sync-destination-postgres' + +export const DEFAULT_STORAGE_SCHEMA = 'stripe' + +export type StoredObject = { + tableName: string + payload: Record +} + +export async function ensureSchema( + pool: pg.Pool, + schema: string = DEFAULT_STORAGE_SCHEMA +): Promise { + const q = quoteIdentifier + await pool.query(`CREATE SCHEMA IF NOT EXISTS ${q(schema)}`) + await pool.query(` + CREATE OR REPLACE FUNCTION ${q(schema)}.set_updated_at() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW := jsonb_populate_record( + NEW, + jsonb_build_object('updated_at', now(), '_updated_at', now()) + ); + RETURN NEW; + END; + $$; + `) +} + +export async function ensureObjectTable( + pool: pg.Pool, + schema: string, + tableName: string, + jsonSchema?: Record +): Promise { + if (jsonSchema) { + const stmts = buildCreateTableWithSchema(schema, tableName, jsonSchema) + for (const stmt of stmts) { + await runSqlAdditive(pool, stmt) + } + return + } + + const q = quoteIdentifier + await pool.query(` + CREATE TABLE IF NOT EXISTS ${q(schema)}.${q(tableName)} ( + "_raw_data" jsonb NOT NULL, + "_last_synced_at" timestamptz, + "_updated_at" timestamptz NOT NULL DEFAULT now(), + "id" text GENERATED ALWAYS AS (("_raw_data"->>'id')::text) STORED, + "created" bigint GENERATED ALWAYS AS (("_raw_data"->>'created')::bigint) STORED, + PRIMARY KEY ("id") + ) + `) + // The fake Stripe server paginates v1 list endpoints by created/id. + await pool.query(` + CREATE INDEX IF NOT EXISTS ${q(`${tableName}_created_id_idx`)} + ON ${q(schema)}.${q(tableName)} ("created" DESC, "id" DESC) + `) +} + +export async function upsertObjects( + pool: pg.Pool, + schema: string, + tableName: string, + objects: Record[] +): Promise { + if (objects.length === 0) return 0 + const q = quoteIdentifier + + const values: unknown[] = [] + const placeholders: string[] = [] + for (const obj of objects) { + values.push(JSON.stringify(obj)) + placeholders.push(`($${values.length}::jsonb)`) + } + + await pool.query( + ` + INSERT INTO ${q(schema)}.${q(tableName)} ("_raw_data") + VALUES ${placeholders.join(', ')} + ON CONFLICT ("id") + DO UPDATE SET + "_raw_data" = EXCLUDED."_raw_data", + "_updated_at" = now() + `, + values + ) + + return objects.length +} + +export function quoteIdentifier(identifier: string): string { + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(identifier)) { + throw new Error(`Invalid SQL identifier "${identifier}"`) + } + return `"${identifier}"` +} + +export function redactConnectionString(url: string): string { + try { + const parsed = new URL(url) + if (parsed.password) parsed.password = '***' + return parsed.toString() + } catch { + return url.replace(/:[^:@]+@/, ':***@') + } +} diff --git a/packages/test-utils/src/index.ts b/packages/test-utils/src/index.ts new file mode 100644 index 000000000..67d2a984f --- /dev/null +++ b/packages/test-utils/src/index.ts @@ -0,0 +1,18 @@ +export { createStripeListServer } from './server/createStripeListServer.js' +export type { StripeListServer, StripeListServerOptions } from './server/types.js' + +export { resolveEndpointSet } from './openapi/endpoints.js' +export type { EndpointDefinition, ResolvedEndpointSet } from './openapi/endpoints.js' + +export { applyCreatedTimestampRange } from './seed/createdTimestamps.js' +export { + DEFAULT_STORAGE_SCHEMA, + ensureSchema, + ensureObjectTable, + upsertObjects, + quoteIdentifier, + redactConnectionString, +} from './db/storage.js' + +export { startDockerPostgres18 } from './postgres/dockerPostgres18.js' +export type { DockerPostgres18Handle } from './postgres/dockerPostgres18.js' diff --git a/packages/test-utils/src/openapi/endpoints.ts b/packages/test-utils/src/openapi/endpoints.ts new file mode 100644 index 000000000..77171279e --- /dev/null +++ b/packages/test-utils/src/openapi/endpoints.ts @@ -0,0 +1,147 @@ +import { + BUNDLED_API_VERSION, + discoverListEndpoints, + isV2Path, + resolveOpenApiSpec, + SpecParser, + OPENAPI_RESOURCE_TABLE_ALIASES, + parsedTableToJsonSchema, + type ListEndpoint, + type OpenApiOperationObject, + type OpenApiSchemaObject, + type OpenApiSchemaOrReference, + type OpenApiSpec, +} from '@stripe/sync-openapi' + +const SCHEMA_REF_PREFIX = '#/components/schemas/' + +export type EndpointQueryParam = { + name: string + required: boolean + schema?: OpenApiSchemaObject +} + +export type EndpointDefinition = ListEndpoint & { + isV2: boolean + queryParams: EndpointQueryParam[] + jsonSchema?: Record +} + +export type ResolvedEndpointSet = { + apiVersion: string + spec: OpenApiSpec + endpoints: Map +} + +export async function resolveEndpointSet(options: { + apiVersion?: string + openApiSpecPath?: string + cacheDir?: string + fetchImpl?: typeof globalThis.fetch +}): Promise { + const apiVersion = options.apiVersion ?? BUNDLED_API_VERSION + const fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis) + const resolved = await resolveOpenApiSpec( + { + apiVersion, + openApiSpecPath: options.openApiSpecPath, + cacheDir: options.cacheDir, + }, + fetchImpl + ) + + const discovered = discoverListEndpoints(resolved.spec) + + const jsonSchemaMap = new Map>() + try { + const parser = new SpecParser() + const parsed = parser.parse(resolved.spec, { + resourceAliases: OPENAPI_RESOURCE_TABLE_ALIASES, + }) + for (const table of parsed.tables) { + jsonSchemaMap.set(table.tableName, parsedTableToJsonSchema(table)) + } + } catch { + // fall through without schemas when the spec can't be parsed + } + + const endpoints = new Map() + + for (const [tableName, endpoint] of discovered.entries()) { + const operation = resolved.spec.paths?.[endpoint.apiPath]?.get + endpoints.set(tableName, { + ...endpoint, + isV2: isV2Path(endpoint.apiPath), + queryParams: extractQueryParams(operation, resolved.spec), + jsonSchema: jsonSchemaMap.get(tableName), + }) + } + + return { + apiVersion: resolved.apiVersion, + spec: resolved.spec, + endpoints, + } +} + +function extractQueryParams( + operation: OpenApiOperationObject | undefined, + spec: OpenApiSpec +): EndpointQueryParam[] { + if (!operation?.parameters) return [] + const params: EndpointQueryParam[] = [] + for (const param of operation.parameters) { + if (param.in !== 'query' || !param.name) continue + const schema = param.schema ? resolveSchema(param.schema, spec) : undefined + params.push({ + name: param.name, + required: Boolean(param.required), + schema, + }) + } + return params +} + +function resolveSchema( + schemaOrRef: OpenApiSchemaOrReference, + spec: OpenApiSpec, + seenRefs = new Set() +): OpenApiSchemaObject | undefined { + if ('$ref' in schemaOrRef) { + if (!schemaOrRef.$ref.startsWith(SCHEMA_REF_PREFIX)) return undefined + if (seenRefs.has(schemaOrRef.$ref)) return undefined + seenRefs.add(schemaOrRef.$ref) + const schemaName = schemaOrRef.$ref.slice(SCHEMA_REF_PREFIX.length) + const resolved = spec.components?.schemas?.[schemaName] + if (!resolved) return undefined + return resolveSchema(resolved, spec, seenRefs) + } + + const schema: OpenApiSchemaObject = { ...schemaOrRef } + if (schema.properties) { + const nextProperties: Record = {} + for (const [key, value] of Object.entries(schema.properties)) { + nextProperties[key] = resolveSchema(value, spec, new Set(seenRefs)) ?? value + } + schema.properties = nextProperties + } + if (schema.items) { + schema.items = resolveSchema(schema.items, spec, new Set(seenRefs)) ?? schema.items + } + if (schema.oneOf) { + schema.oneOf = schema.oneOf.map( + (candidate) => resolveSchema(candidate, spec, new Set(seenRefs)) ?? candidate + ) + } + if (schema.anyOf) { + schema.anyOf = schema.anyOf.map( + (candidate) => resolveSchema(candidate, spec, new Set(seenRefs)) ?? candidate + ) + } + if (schema.allOf) { + schema.allOf = schema.allOf.map( + (candidate) => resolveSchema(candidate, spec, new Set(seenRefs)) ?? candidate + ) + } + return schema +} diff --git a/packages/test-utils/src/openapi/filters.ts b/packages/test-utils/src/openapi/filters.ts new file mode 100644 index 000000000..ea58e89fe --- /dev/null +++ b/packages/test-utils/src/openapi/filters.ts @@ -0,0 +1,165 @@ +import type { EndpointQueryParam } from './endpoints.js' +import type { OpenApiSchemaObject, OpenApiSchemaOrReference } from '@stripe/sync-openapi' + +const NESTED_QUERY_KEY = /^([^\[]+)\[([^\]]+)\]$/ + +export type ValidatedQuery = { + ok: true + forward: URLSearchParams +} + +export type QueryValidationError = { + ok: false + statusCode: number + message: string + details: string[] + allowed: string[] +} + +export function validateQueryAgainstOpenApi( + input: URLSearchParams, + params: EndpointQueryParam[] +): ValidatedQuery | QueryValidationError { + const forward = new URLSearchParams() + const errors: string[] = [] + const provided = new Set() + const directAllowed = new Set() + const objectAllowed = new Map>() + const paramByName = new Map() + + for (const param of params) { + paramByName.set(param.name, param) + directAllowed.add(param.name) + const objectProps = objectPropertyNames(param.schema) + if (objectProps.size > 0) { + objectAllowed.set(param.name, objectProps) + } + } + + for (const [key, value] of input.entries()) { + const direct = paramByName.get(key) + if (direct) { + if (!isValidForSchema(value, direct.schema)) { + errors.push(`Invalid value for query parameter "${key}"`) + continue + } + forward.append(key, value) + provided.add(key) + continue + } + + const nested = key.match(NESTED_QUERY_KEY) + if (!nested) { + errors.push(`Unknown query parameter "${key}"`) + continue + } + + const [, base, subKey] = nested + const nestedAllowed = objectAllowed.get(base) + if (!nestedAllowed || !nestedAllowed.has(subKey)) { + errors.push(`Unknown nested query parameter "${key}"`) + continue + } + + const baseSchema = paramByName.get(base)?.schema + const objectVariant = findObjectVariant(baseSchema) + const propertySchema = + objectVariant && objectVariant.properties + ? resolvePropertySchema(objectVariant.properties[subKey]) + : undefined + if (!isValidForSchema(value, propertySchema)) { + errors.push(`Invalid value for query parameter "${key}"`) + continue + } + + forward.append(key, value) + provided.add(base) + } + + for (const param of params) { + if (param.required && !provided.has(param.name)) { + errors.push(`Missing required query parameter "${param.name}"`) + } + } + + const allowed = [...directAllowed].sort() + for (const [name, props] of objectAllowed.entries()) { + for (const prop of props) allowed.push(`${name}[${prop}]`) + } + allowed.sort() + + if (errors.length > 0) { + return { + ok: false, + statusCode: 400, + message: 'Query parameters do not match OpenAPI definition', + details: errors, + allowed, + } + } + + return { ok: true, forward } +} + +function objectPropertyNames(schema: OpenApiSchemaObject | undefined): Set { + if (!schema) return new Set() + if (schema.type === 'object' && schema.properties) { + return new Set(Object.keys(schema.properties)) + } + for (const variant of [...(schema.anyOf ?? []), ...(schema.oneOf ?? [])]) { + if ('$ref' in variant) continue + const names = objectPropertyNames(variant) + if (names.size > 0) return names + } + return new Set() +} + +function findObjectVariant( + schema: OpenApiSchemaObject | undefined +): OpenApiSchemaObject | undefined { + if (!schema) return undefined + if (schema.type === 'object' && schema.properties) return schema + for (const variant of [...(schema.anyOf ?? []), ...(schema.oneOf ?? [])]) { + if ('$ref' in variant) continue + const found = findObjectVariant(variant) + if (found) return found + } + return undefined +} + +function resolvePropertySchema( + schema: OpenApiSchemaOrReference | undefined +): OpenApiSchemaObject | undefined { + if (!schema || '$ref' in schema) return undefined + return schema +} + +function isValidForSchema(value: string, schema: OpenApiSchemaObject | undefined): boolean { + if (!schema) return true + + if (schema.enum && schema.enum.length > 0) { + return schema.enum.some((entry) => String(entry) === value) + } + + if (schema.oneOf?.length) { + return schema.oneOf.some((candidate) => + isValidForSchema(value, resolvePropertySchema(candidate)) + ) + } + if (schema.anyOf?.length) { + return schema.anyOf.some((candidate) => + isValidForSchema(value, resolvePropertySchema(candidate)) + ) + } + + switch (schema.type) { + case 'integer': + return /^-?\d+$/.test(value) + case 'number': + return Number.isFinite(Number(value)) + case 'boolean': + return value === 'true' || value === 'false' + default: + return true + } +} diff --git a/packages/test-utils/src/postgres/dockerPostgres18.ts b/packages/test-utils/src/postgres/dockerPostgres18.ts new file mode 100644 index 000000000..19126a792 --- /dev/null +++ b/packages/test-utils/src/postgres/dockerPostgres18.ts @@ -0,0 +1,105 @@ +import { execSync, exec } from 'node:child_process' +import pg from 'pg' + +export type DockerPostgres18Handle = { + containerId: string + hostPort: number + connectionString: string + stop: () => Promise +} + +export async function startDockerPostgres18(): Promise { + try { + execSync('docker info', { stdio: 'ignore' }) + } catch { + throw new Error('Docker is not running. Start Docker and try again.') + } + + let containerId: string + try { + containerId = execSync( + [ + 'docker run -d --rm -p 0:5432', + '-e POSTGRES_PASSWORD=postgres', + '-e POSTGRES_DB=postgres', + 'postgres:18', + '-c ssl=on', + '-c ssl_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem', + '-c ssl_key_file=/etc/ssl/private/ssl-cert-snakeoil.key', + ].join(' '), + { encoding: 'utf8' } + ).trim() + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + throw new Error(`Failed to start postgres:18 container: ${msg}`) + } + + const hostPortValue = execSync(`docker port ${containerId} 5432`, { + encoding: 'utf8', + }) + .trim() + .split(':') + .pop() + + if (!hostPortValue) { + await stopContainer(containerId) + throw new Error(`Failed to determine mapped host port for postgres container ${containerId}`) + } + const hostPort = Number(hostPortValue) + if (!Number.isFinite(hostPort)) { + await stopContainer(containerId) + throw new Error(`Invalid mapped host port "${hostPortValue}" for postgres container ${containerId}`) + } + + const connectionString = `postgresql://postgres:postgres@localhost:${hostPort}/postgres` + await waitForPostgres(connectionString) + + let stopped = false + const cleanupOnExit = () => { + if (stopped) return + stopped = true + try { + execSync(`docker rm -fv ${containerId}`, { stdio: 'ignore' }) + } catch {} + } + process.once('exit', cleanupOnExit) + + return { + containerId, + hostPort, + connectionString, + stop: async () => { + if (stopped) return + stopped = true + process.off('exit', cleanupOnExit) + await stopContainer(containerId) + }, + } +} + +async function waitForPostgres(connectionString: string): Promise { + const pool = new pg.Pool({ connectionString }) + try { + for (let i = 0; i < 60; i++) { + try { + await pool.query('SELECT 1') + return + } catch { + await sleep(500) + } + } + throw new Error('Postgres container did not become ready in time') + } finally { + await pool.end().catch(() => undefined) + } +} + +function stopContainer(containerId: string): Promise { + return new Promise((resolve) => { + exec(`docker rm -fv ${containerId}`, () => resolve()) + }) +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/packages/test-utils/src/seed/createdTimestamps.ts b/packages/test-utils/src/seed/createdTimestamps.ts new file mode 100644 index 000000000..3bfcc7c86 --- /dev/null +++ b/packages/test-utils/src/seed/createdTimestamps.ts @@ -0,0 +1,27 @@ +export type CreatedTimestampRange = { + startUnix: number + endUnix: number +} + +export function applyCreatedTimestampRange( + objects: Record[], + range: CreatedTimestampRange | undefined +): Record[] { + if (!range) return objects + if (objects.length === 0) return objects + + // Max created is endUnix - 1 so no object lands on the boundary. + // Matches Stripe's created[gte]/created[lt] semantics. + const maxCreated = range.endUnix - 1 + + if (objects.length === 1) { + return [{ ...objects[0], created: maxCreated }] + } + + const span = Math.max(0, maxCreated - range.startUnix) + return objects.map((object, index) => { + const ratio = index / (objects.length - 1) + const created = range.startUnix + Math.floor(span * ratio) + return { ...object, created } + }) +} diff --git a/packages/test-utils/src/server/createStripeListServer.ts b/packages/test-utils/src/server/createStripeListServer.ts new file mode 100644 index 000000000..bf1d1de6f --- /dev/null +++ b/packages/test-utils/src/server/createStripeListServer.ts @@ -0,0 +1,724 @@ +import { Hono } from 'hono' +import type { Context } from 'hono' +import { serve } from '@hono/node-server' +import type { ServerType } from '@hono/node-server' +import pg from 'pg' +import { + DEFAULT_STORAGE_SCHEMA, + ensureSchema, + quoteIdentifier, + redactConnectionString, +} from '../db/storage.js' +import { resolveEndpointSet, type EndpointDefinition } from '../openapi/endpoints.js' +import { validateQueryAgainstOpenApi } from '../openapi/filters.js' +import { startDockerPostgres18, type DockerPostgres18Handle } from '../postgres/dockerPostgres18.js' +import type { + StripeListServerOptions, + StripeListServer, + StripeListServerAuthOptions, + StripeListServerFailureRule, + PageResult, + V1PageQuery, + V2PageQuery, +} from './types.js' +export type { StripeListServerOptions, StripeListServer } from './types.js' + +// ── Helpers ─────────────────────────────────────────────────────── + +const V2_PAGE_CURSOR_QUERY_PARAM = 'page' + +function makeFakeAccount(created: number) { + return { + id: 'acct_test_fake_000000', + object: 'account', + type: 'standard', + charges_enabled: true, + payouts_enabled: true, + details_submitted: true, + business_type: 'company', + country: 'US', + default_currency: 'usd', + email: 'test@example.com', + created, + settings: { dashboard: { display_name: 'Test Account' } }, + } +} + +// ── Server factory ──────────────────────────────────────────────── + +export async function createStripeListServer( + options: StripeListServerOptions = {} +): Promise { + const fetchImpl = options.fetchImpl ?? globalThis.fetch.bind(globalThis) + const schema = options.schema ?? DEFAULT_STORAGE_SCHEMA + const endpointSet = await resolveEndpointSet({ + apiVersion: options.apiVersion, + openApiSpecPath: options.openApiSpecPath, + fetchImpl, + }) + + let dockerHandle: DockerPostgres18Handle | undefined + let postgresMode: 'docker' | 'external' = 'external' + const postgresUrl = options.postgresUrl ?? process.env.POSTGRES_URL + if (!postgresUrl) { + dockerHandle = await startDockerPostgres18() + postgresMode = 'docker' + } + const connectionString = postgresUrl ?? dockerHandle?.connectionString + if (!connectionString) { + throw new Error('No Postgres connection string available') + } + + const pool = new pg.Pool({ connectionString }) + await ensureSchema(pool, schema) + + const fakeAccount = makeFakeAccount(options.accountCreated ?? Math.floor(Date.now() / 1000)) + const failureStates = (options.failures ?? []).map(() => ({ matches: 0, failures: 0 })) + const logRequests = options.logRequests ?? true + + // ── Build Hono app ──────────────────────────────────────────── + + const app = new Hono() + + const perPath = new Map() + + app.use('*', async (c, next) => { + const start = performance.now() + await next() + const elapsed = performance.now() - start + const stats = perPath.get(c.req.path) + if (stats) { + stats.count++ + stats.totalMs += elapsed + if (elapsed > stats.maxMs) stats.maxMs = elapsed + } else { + perPath.set(c.req.path, { count: 1, totalMs: elapsed, maxMs: elapsed }) + } + if (logRequests) { + logRequest(c.req.method, c.req.path, c.res.status) + } + }) + + for (const prefix of ['/v1/*', '/v2/*'] as const) { + app.use(prefix, async (c, next) => { + const intercepted = maybeInterceptStripeApiRequest( + c, + options.auth, + options.failures ?? [], + failureStates + ) + if (intercepted) return intercepted + await next() + }) + } + + app.get('/health', (c) => + c.json({ + ok: true, + api_version: endpointSet.apiVersion, + endpoint_count: endpointSet.endpoints.size, + }) + ) + + app.get('/db-health', async (c) => { + const probe = await pool.query('SELECT 1 AS ok') + return c.json({ + ok: probe.rows[0]?.ok === 1, + postgres_mode: postgresMode, + postgres_url: redactConnectionString(connectionString), + schema, + }) + }) + + app.get('/v1/account', (c) => c.json(fakeAccount)) + + for (const ep of endpointSet.endpoints.values()) { + app.get(ep.apiPath, (c) => + handleList(c, pool, schema, ep, options.validateQueryParams ?? false) + ) + app.get(`${ep.apiPath}/:id`, (c) => handleRetrieve(c, pool, schema, ep, c.req.param('id'))) + } + + for (const prefix of ['/v1/*', '/v2/*'] as const) { + app.all(prefix, (c) => { + if (c.req.method !== 'GET') { + return c.json( + { error: { type: 'invalid_request_error', message: 'Method not allowed' } }, + 405 + ) + } + return c.json( + { + error: { + type: 'invalid_request_error', + message: `Unrecognized request URL (GET: ${c.req.path})`, + }, + }, + 404 + ) + }) + } + + app.onError((err, c) => { + const message = err instanceof Error ? err.message : String(err) + return c.json({ error: message }, 500) + }) + + // ── Start server ────────────────────────────────────────────── + + const serverHost = options.host ?? '127.0.0.1' + const serverPort = options.port ?? 5555 + + let nodeServer: ServerType | undefined + await new Promise((resolve, reject) => { + try { + nodeServer = serve({ fetch: app.fetch, port: serverPort, hostname: serverHost }, () => + resolve() + ) + } catch (err) { + reject(err) + } + }) + + const addr = nodeServer!.address() + const actualPort = typeof addr === 'object' && addr ? addr.port : serverPort + + let closed = false + const close = async (): Promise => { + if (closed) return + closed = true + if (perPath.size > 0) { + const totalReqs = [...perPath.values()].reduce((a, b) => a + b.count, 0) + const totalMs = [...perPath.values()].reduce((a, b) => a + b.totalMs, 0) + const sorted = [...perPath.entries()].sort((a, b) => b[1].totalMs - a[1].totalMs) + process.stderr.write( + `[test-server] ${totalReqs} reqs across ${perPath.size} endpoints, total_time=${(totalMs / 1000).toFixed(1)}s\n` + ) + for (const [p, s] of sorted) { + const avg = (s.totalMs / s.count).toFixed(1) + const total = (s.totalMs / 1000).toFixed(1) + process.stderr.write( + ` ${total.padStart(6)}s total ${s.count.toString().padStart(5)} reqs ${avg.padStart(6)}ms avg ${s.maxMs.toFixed(0).padStart(5)}ms max ${p}\n` + ) + } + } + if (nodeServer) { + await new Promise((resolve) => { + nodeServer!.close(() => resolve()) + }) + } + await pool.end().catch(() => undefined) + if (dockerHandle) await dockerHandle.stop() + } + + const cleanup = () => { + void close() + } + process.once('SIGINT', cleanup) + process.once('SIGTERM', cleanup) + + return { + host: serverHost, + port: actualPort, + url: `http://${serverHost}:${actualPort}`, + postgresUrl: connectionString, + postgresMode, + close, + } +} + +// --------------------------------------------------------------------------- +// List — paginated read from Postgres, returns Stripe list response format +// --------------------------------------------------------------------------- + +async function handleList( + c: Context, + pool: pg.Pool, + schema: string, + endpoint: EndpointDefinition, + validateQueryParams: boolean +): Promise { + const query = new URL(c.req.url).searchParams + if (validateQueryParams) { + const validated = validateQueryAgainstOpenApi( + stripInternalPaginationParams(query, endpoint), + endpoint.queryParams + ) + if (!validated.ok) { + process.stderr.write( + `[sync-test-utils] query validation failed for ${endpoint.apiPath}: ` + + `query="${query.toString()}" details=${JSON.stringify(validated.details)} ` + + `allowed=${JSON.stringify(validated.allowed)}\n` + ) + return c.json( + { + error: { + type: 'invalid_request_error', + message: validated.message, + details: validated.details, + allowed: validated.allowed, + }, + }, + validated.statusCode as 400 + ) + } + } + + if (endpoint.isV2) { + const limit = clampLimit(query.get('limit') ?? undefined, 20) + const pageToken = query.get(V2_PAGE_CURSOR_QUERY_PARAM) ?? undefined + const afterId = pageToken ? decodePageToken(pageToken) : undefined + + const { data, hasMore, lastId } = await queryPageV2(pool, schema, endpoint.tableName, { + limit, + afterId, + createdGt: parseTimestampParam(query.get('created[gt]') ?? undefined), + createdGte: parseTimestampParam(query.get('created[gte]') ?? undefined), + createdLt: parseTimestampParam(query.get('created[lt]') ?? undefined), + createdLte: parseTimestampParam(query.get('created[lte]') ?? undefined), + }) + + const nextPageUrl = + hasMore && lastId + ? buildV2NextPageUrl( + endpoint.apiPath, + limit, + encodePageToken(lastId), + new URL(c.req.url).searchParams + ) + : null + + return c.json({ + data, + next_page_url: nextPageUrl, + previous_page_url: null, + }) + } + + const limit = clampLimit(query.get('limit') ?? undefined, 10) + const v1Query = { + limit, + afterId: query.get('starting_after') ?? undefined, + beforeId: query.get('ending_before') ?? undefined, + createdGt: parseIntParam(query.get('created[gt]') ?? undefined), + createdGte: parseIntParam(query.get('created[gte]') ?? undefined), + createdLt: parseIntParam(query.get('created[lt]') ?? undefined), + createdLte: parseIntParam(query.get('created[lte]') ?? undefined), + } + const supportsForwardPagination = endpoint.queryParams.some( + (param) => param.name === 'starting_after' + ) + const { data, hasMore } = supportsForwardPagination + ? await queryPageV1(pool, schema, endpoint.tableName, v1Query) + : await queryAllV1(pool, schema, endpoint.tableName, v1Query) + + return c.json({ + object: 'list', + url: endpoint.apiPath, + has_more: hasMore, + data, + }) +} + +// --------------------------------------------------------------------------- +// Retrieve — single object by ID from Postgres +// --------------------------------------------------------------------------- + +async function handleRetrieve( + c: Context, + pool: pg.Pool, + schema: string, + endpoint: EndpointDefinition, + objectId: string +): Promise { + let rows: { _raw_data: Record }[] + try { + const result = await pool.query( + `SELECT _raw_data FROM ${quoteIdentifier(schema)}.${quoteIdentifier(endpoint.tableName)} WHERE id = $1 LIMIT 1`, + [objectId] + ) + rows = result.rows + } catch (err: unknown) { + if ((err as { code?: string })?.code === '42P01') { + rows = [] + } else { + throw err + } + } + + if (rows.length === 0) { + return c.json( + { + error: { + type: 'invalid_request_error', + message: `No such ${endpoint.resourceId}: '${objectId}'`, + param: 'id', + code: 'resource_missing', + }, + }, + 404 + ) + } + + return c.json(rows[0]._raw_data as Record) +} + +// --------------------------------------------------------------------------- +// Postgres queries — paginated reads from seeded tables +// --------------------------------------------------------------------------- + +/** + * V1: created DESC, id DESC; tuple cursors for starting_after / ending_before. + * Cursor lookups are inlined as subqueries to avoid extra round trips. + */ +async function queryPageV1( + pool: pg.Pool, + schema: string, + tableName: string, + opts: V1PageQuery +): Promise { + const conditions: string[] = [] + const values: unknown[] = [] + let idx = 0 + const useEndingBefore = !opts.afterId && !!opts.beforeId + const table = `${quoteIdentifier(schema)}.${quoteIdentifier(tableName)}` + + if (opts.afterId) { + conditions.push( + `(created, id) < ((SELECT created FROM ${table} WHERE id = $${++idx}), $${idx})` + ) + values.push(opts.afterId) + } + if (opts.beforeId) { + conditions.push( + `(created, id) > ((SELECT created FROM ${table} WHERE id = $${++idx}), $${idx})` + ) + values.push(opts.beforeId) + } + if (opts.createdGt != null) { + conditions.push(`created > $${++idx}`) + values.push(opts.createdGt) + } + if (opts.createdGte != null) { + conditions.push(`created >= $${++idx}`) + values.push(opts.createdGte) + } + if (opts.createdLt != null) { + conditions.push(`created < $${++idx}`) + values.push(opts.createdLt) + } + if (opts.createdLte != null) { + conditions.push(`created <= $${++idx}`) + values.push(opts.createdLte) + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' + const fetchLimit = opts.limit + 1 + values.push(fetchLimit) + + const orderDir = useEndingBefore ? 'ASC' : 'DESC' + const orderClause = `ORDER BY created ${orderDir}, id ${orderDir}` + + const rows = await safeQuery( + pool, + `SELECT _raw_data FROM ${table} ${where} ${orderClause} LIMIT $${++idx}`, + values, + tableName + ) + + const hasMore = rows.length > opts.limit + const page = rows.slice(0, opts.limit) + if (useEndingBefore) page.reverse() + + const data = page.map((r) => r._raw_data) + const lastId = data.length > 0 ? (data[data.length - 1].id as string) : undefined + return { data, hasMore, lastId } +} + +async function queryAllV1( + pool: pg.Pool, + schema: string, + tableName: string, + opts: Omit +): Promise<{ data: Record[]; hasMore: false }> { + const conditions: string[] = [] + const values: unknown[] = [] + let idx = 0 + + if (opts.createdGt != null) { + conditions.push(`created > $${++idx}`) + values.push(opts.createdGt) + } + if (opts.createdGte != null) { + conditions.push(`created >= $${++idx}`) + values.push(opts.createdGte) + } + if (opts.createdLt != null) { + conditions.push(`created < $${++idx}`) + values.push(opts.createdLt) + } + if (opts.createdLte != null) { + conditions.push(`created <= $${++idx}`) + values.push(opts.createdLte) + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' + const table = `${quoteIdentifier(schema)}.${quoteIdentifier(tableName)}` + const rows = await safeQuery( + pool, + `SELECT _raw_data FROM ${table} ${where} ORDER BY created DESC, id DESC`, + values, + tableName + ) + + return { data: rows.map((row) => row._raw_data), hasMore: false } +} + +/** + * V2: opaque page tokens map to id ASC + `id > cursor`. + * When the endpoint supports `created`, we apply the created window too. + */ +async function queryPageV2( + pool: pg.Pool, + schema: string, + tableName: string, + opts: V2PageQuery +): Promise { + const conditions: string[] = [] + const values: unknown[] = [] + let idx = 0 + + if (opts.afterId) { + conditions.push(`id > $${++idx}`) + values.push(opts.afterId) + } + if (opts.createdGt != null) { + conditions.push(`created > $${++idx}`) + values.push(opts.createdGt) + } + if (opts.createdGte != null) { + conditions.push(`created >= $${++idx}`) + values.push(opts.createdGte) + } + if (opts.createdLt != null) { + conditions.push(`created < $${++idx}`) + values.push(opts.createdLt) + } + if (opts.createdLte != null) { + conditions.push(`created <= $${++idx}`) + values.push(opts.createdLte) + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '' + const fetchLimit = opts.limit + 1 + values.push(fetchLimit) + + const table = `${quoteIdentifier(schema)}.${quoteIdentifier(tableName)}` + const rows = await safeQuery( + pool, + `SELECT _raw_data FROM ${table} ${where} ORDER BY id ASC LIMIT $${++idx}`, + values, + tableName + ) + + const hasMore = rows.length > opts.limit + const page = rows.slice(0, opts.limit) + const data = page.map((r) => r._raw_data) + const lastId = data.length > 0 ? (data[data.length - 1].id as string) : undefined + return { data, hasMore, lastId } +} + +async function safeQuery( + pool: pg.Pool, + sql: string, + values: unknown[], + tableName: string +): Promise<{ _raw_data: Record }[]> { + try { + const result = await pool.query(sql, values) + return result.rows + } catch (err: unknown) { + if ((err as { code?: string })?.code === '42P01') { + process.stderr.write( + `[sync-test-utils] WARNING: table "${tableName}" does not exist — returning empty result. Was the database seeded?\n` + ) + return [] + } + throw err + } +} + +// --------------------------------------------------------------------------- +// Utilities +// --------------------------------------------------------------------------- + +function clampLimit(raw: string | undefined, defaultLimit: number): number { + if (raw == null) return defaultLimit + const n = parseInt(raw, 10) + if (!Number.isFinite(n) || n < 1) return defaultLimit + return Math.min(n, 100) +} + +function parseIntParam(raw: string | undefined): number | undefined { + if (raw == null) return undefined + const n = parseInt(raw, 10) + return Number.isFinite(n) ? n : undefined +} + +function parseTimestampParam(raw: string | undefined): number | undefined { + if (raw == null) return undefined + if (/^-?\d+$/.test(raw)) return parseInt(raw, 10) + const parsed = Date.parse(raw) + return Number.isFinite(parsed) ? Math.floor(parsed / 1000) : undefined +} + +function encodePageToken(id: string): string { + return Buffer.from(id).toString('base64url') +} + +function decodePageToken(token: string): string { + return Buffer.from(token, 'base64url').toString() +} + +function stripInternalPaginationParams( + query: URLSearchParams, + endpoint: EndpointDefinition +): URLSearchParams { + const normalized = new URLSearchParams(query) + if (endpoint.isV2) { + normalized.delete(V2_PAGE_CURSOR_QUERY_PARAM) + } + return normalized +} + +/** Carry forward incoming filters on v2 next_page_url. */ +function buildV2NextPageUrl( + apiPath: string, + limit: number, + pageToken: string, + incoming: URLSearchParams +): string { + const qs = new URLSearchParams() + qs.set('limit', String(limit)) + qs.set(V2_PAGE_CURSOR_QUERY_PARAM, pageToken) + for (const [key, value] of incoming.entries()) { + if (key === 'limit' || key === V2_PAGE_CURSOR_QUERY_PARAM) continue + qs.append(key, value) + } + return `${apiPath}?${qs.toString()}` +} + +function logRequest(method: string, path: string, statusCode: number): void { + process.stderr.write(`[sync-test-utils] ${method} ${path} → ${statusCode}\n`) +} + +function maybeInterceptStripeApiRequest( + c: Context, + auth: StripeListServerAuthOptions | undefined, + failures: StripeListServerFailureRule[], + failureStates: Array<{ matches: number; failures: number }> +): Response | undefined { + const authFailure = maybeHandleAuthFailure(c, auth) + if (authFailure) return authFailure + + return maybeHandleInjectedFailure(c, failures, failureStates) +} + +function maybeHandleAuthFailure( + c: Context, + auth: StripeListServerAuthOptions | undefined +): Response | undefined { + if (!auth) return undefined + const protectedPaths = auth.protectedPaths ?? ['/v1/*', '/v2/*'] + if (!pathMatchesAny(c.req.path, protectedPaths)) return undefined + + const bearerToken = extractBearerToken(c.req.header('authorization')) + if (bearerToken === auth.expectedBearerToken) return undefined + + return c.json( + { + error: { + type: 'invalid_request_error', + message: + auth.errorMessage ?? + (bearerToken ? `Invalid API Key provided: ${bearerToken}` : 'Invalid API Key provided'), + }, + }, + 401 + ) +} + +function maybeHandleInjectedFailure( + c: Context, + failures: StripeListServerFailureRule[], + failureStates: Array<{ matches: number; failures: number }> +): Response | undefined { + for (const [index, rule] of failures.entries()) { + if (!matchesFailureRule(c.req.method, c.req.path, rule)) continue + + const state = failureStates[index]! + state.matches += 1 + + const after = rule.after ?? 0 + const times = rule.times ?? Number.POSITIVE_INFINITY + if (state.matches <= after || state.failures >= times) continue + + state.failures += 1 + return new Response(JSON.stringify(buildFailureBody(rule, c.req.method, c.req.path)), { + status: rule.status, + headers: { 'Content-Type': 'application/json' }, + }) + } + + return undefined +} + +function matchesFailureRule( + method: string, + path: string, + rule: StripeListServerFailureRule +): boolean { + const expectedMethod = (rule.method ?? 'GET').toUpperCase() + if (method.toUpperCase() !== expectedMethod) return false + return matchesPathPattern(path, rule.path) +} + +function buildFailureBody( + rule: StripeListServerFailureRule, + method: string, + path: string +): Record { + if (rule.body) return rule.body + if (rule.stripeError) { + return { + error: { + type: rule.stripeError.type ?? 'api_error', + message: rule.stripeError.message, + ...(rule.stripeError.code ? { code: rule.stripeError.code } : {}), + }, + } + } + return { + error: { + type: 'api_error', + message: `Injected failure for ${method.toUpperCase()} ${path}`, + }, + } +} + +function pathMatchesAny(path: string, patterns: string[]): boolean { + return patterns.some((pattern) => matchesPathPattern(path, pattern)) +} + +function matchesPathPattern(path: string, pattern: string): boolean { + if (pattern.endsWith('*')) { + return path.startsWith(pattern.slice(0, -1)) + } + return path === pattern +} + +function extractBearerToken(header: string | undefined): string | null { + if (!header) return null + const match = /^Bearer\s+(.+)$/i.exec(header.trim()) + return match?.[1] ?? null +} diff --git a/packages/test-utils/src/server/types.ts b/packages/test-utils/src/server/types.ts new file mode 100644 index 000000000..0e2537c9c --- /dev/null +++ b/packages/test-utils/src/server/types.ts @@ -0,0 +1,91 @@ +export type StripeListServerOptions = { + port?: number + host?: string + /** Whether to emit per-request logs to stderr. @default true */ + logRequests?: boolean + /** Whether to validate incoming list query params against OpenAPI metadata. @default false */ + validateQueryParams?: boolean + apiVersion?: string + openApiSpecPath?: string + postgresUrl?: string + schema?: string + /** Unix timestamp for the fake account's `created` field. Controls backfill range start. */ + accountCreated?: number + fetchImpl?: typeof globalThis.fetch + /** Optional auth guard for Stripe API routes. */ + auth?: StripeListServerAuthOptions + /** Optional injected failures for specific Stripe API routes. */ + failures?: StripeListServerFailureRule[] +} + +export type StripeListServerAuthOptions = { + /** Expected bearer token value (without the `Bearer ` prefix). */ + expectedBearerToken: string + /** + * Route patterns to protect. Defaults to all Stripe API routes (`/v1/*`, `/v2/*`). + * Supports exact paths (e.g. `/v1/account`) and prefix globs ending in `*`. + */ + protectedPaths?: string[] + /** Override the Stripe-style error message returned for auth failures. */ + errorMessage?: string +} + +export type StripeListServerFailureRule = { + /** Exact route path or prefix glob ending in `*` (e.g. `/v1/customers` or `/v1/*`). */ + path: string + /** HTTP method to match. Defaults to `GET`. */ + method?: string + /** Response status code to return when this rule triggers. */ + status: number + /** + * Allow this many matching requests through before starting to fail. + * Example: `after: 1` fails the second matching request. + */ + after?: number + /** + * Number of matching requests to fail after the `after` threshold is reached. + * Defaults to unlimited. + */ + times?: number + /** + * Stripe-style error payload. When omitted, a generic error body is generated. + * Returned as `{ error: ... }`. + */ + stripeError?: { + type?: string + message: string + code?: string + } + /** Raw JSON body override. Takes precedence over `stripeError` when set. */ + body?: Record +} + +export type StripeListServer = { + host: string + port: number + url: string + postgresUrl: string + postgresMode: 'docker' | 'external' + close: () => Promise +} + +export type PageResult = { data: Record[]; hasMore: boolean; lastId?: string } + +export type V1PageQuery = { + limit: number + afterId?: string + beforeId?: string + createdGt?: number + createdGte?: number + createdLt?: number + createdLte?: number +} + +export type V2PageQuery = { + limit: number + afterId?: string + createdGt?: number + createdGte?: number + createdLt?: number + createdLte?: number +} diff --git a/packages/test-utils/tsconfig.json b/packages/test-utils/tsconfig.json new file mode 100644 index 000000000..2481fe545 --- /dev/null +++ b/packages/test-utils/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts", "src/**/__tests__/**"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d0aeb115..959ea3f10 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -433,9 +433,15 @@ importers: '@stripe/sync-source-stripe': specifier: workspace:* version: link:../packages/source-stripe + '@stripe/sync-test-utils': + specifier: workspace:* + version: link:../packages/test-utils '@temporalio/client': specifier: ^1 version: 1.15.0 + '@temporalio/testing': + specifier: ^1.15.0 + version: 1.15.0(tslib@2.8.1) '@temporalio/worker': specifier: ^1 version: 1.15.0(tslib@2.8.1) @@ -609,6 +615,34 @@ importers: specifier: ^3.2.4 version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/test-utils: + dependencies: + '@hono/node-server': + specifier: ^1.19.11 + version: 1.19.11(hono@4.12.8) + '@stripe/sync-destination-postgres': + specifier: workspace:* + version: link:../destination-postgres + '@stripe/sync-openapi': + specifier: workspace:* + version: link:../openapi + hono: + specifier: ^4.12.8 + version: 4.12.8 + pg: + specifier: ^8.16.3 + version: 8.16.3 + devDependencies: + '@types/node': + specifier: ^24.10.1 + version: 24.10.1 + '@types/pg': + specifier: ^8.15.5 + version: 8.20.0 + vitest: + specifier: ^3.2.4 + version: 3.2.4(@types/node@24.10.1)(jiti@2.6.1)(lightningcss@1.32.0)(terser@5.46.1)(tsx@4.21.0)(yaml@2.8.1) + packages/ts-cli: dependencies: citty: diff --git a/scripts/generate-openapi-specs.ts b/scripts/generate-openapi-specs.ts index 56969715f..f2feabd07 100644 --- a/scripts/generate-openapi-specs.ts +++ b/scripts/generate-openapi-specs.ts @@ -21,7 +21,7 @@ const resolver = await createConnectorResolver({ sources: { stripe: (sourceStripe as any).default ?? sourceStripe }, destinations: { postgres: (destinationPostgres as any).default ?? destinationPostgres, - 'google_sheets': (destinationGoogleSheets as any).default ?? destinationGoogleSheets, + google_sheets: (destinationGoogleSheets as any).default ?? destinationGoogleSheets, }, })