Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .env.production.example
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ PREVIEW_PROXY_SUBDOMAIN_SUFFIX=preview
# the separate develop soak env file.
# R_APP_ENV=production

# How long an API shutdown lets in-flight Fast turns finish before aborting
# the remainder. Keep it under the platform's SIGTERM-to-SIGKILL grace
# window; 0 aborts active turns immediately.
# R_API_SHUTDOWN_DRAIN_MS=20000

# Replace local defaults before exposing a shared deployment.
DATABASE_URL=postgres://postgres:password@postgres:5432/roomote_development
REDIS_URL=redis://redis:6379
Expand Down
90 changes: 83 additions & 7 deletions apps/api/src/__tests__/graceful-shutdown.test.ts

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 42 additions & 2 deletions apps/api/src/graceful-shutdown.ts
Original file line number Diff line number Diff line change
@@ -1,30 +1,70 @@
import type { ServerType } from '@hono/node-server';
import {
abortActiveFastAgentTurns,
beginFastAgentTurnDrain,
FastAgentProcessShutdownError,
waitForActiveFastAgentTurnsToSettle,
} from '@roomote/cloud-agents/server';

// Most Fast turns finish within seconds, so letting them settle turns a
// deploy-time interruption into a completed answer. The default leaves room
// for the straggler abort, closeout delivery, and Sentry flush inside a
// typical 30s SIGTERM-to-SIGKILL grace window. R_API_SHUTDOWN_DRAIN_MS
// overrides it; 0 restores the previous abort-immediately behavior.
const DEFAULT_API_SHUTDOWN_DRAIN_MS = 20_000;

export function resolveApiShutdownDrainMs(
env: NodeJS.ProcessEnv = process.env,
): number {
const raw = env.R_API_SHUTDOWN_DRAIN_MS?.trim();
if (!raw) return DEFAULT_API_SHUTDOWN_DRAIN_MS;
const parsed = Number(raw);
return Number.isFinite(parsed) && parsed >= 0
? parsed
: DEFAULT_API_SHUTDOWN_DRAIN_MS;
}

type ApiShutdownOptions = {
abortTurns?: typeof abortActiveFastAgentTurns;
beginDrain?: typeof beginFastAgentTurnDrain;
waitForTurns?: typeof waitForActiveFastAgentTurnsToSettle;
drainMs?: number;
exitProcess?: (code?: number) => never;
flushSentry?: () => Promise<unknown>;
logError?: (...args: Parameters<typeof console.error>) => void;
logWarn?: (...args: Parameters<typeof console.warn>) => void;
};

export async function gracefullyShutdownApi(
server: ServerType,
signal: NodeJS.Signals,
{
abortTurns = abortActiveFastAgentTurns,
beginDrain = beginFastAgentTurnDrain,
waitForTurns = waitForActiveFastAgentTurnsToSettle,
drainMs = resolveApiShutdownDrainMs(),
exitProcess = process.exit,
flushSentry = async () => undefined,
logError = (...args) => console.error(...args),
logWarn = (...args) => console.warn(...args),
}: ApiShutdownOptions = {},
): Promise<void> {
const abortPromise = abortTurns(new FastAgentProcessShutdownError(signal));
const closeError = await new Promise<Error | null>((resolve) => {
const reason = new FastAgentProcessShutdownError(signal);
// Refuse new turn admissions and stop accepting connections first, then
// give in-flight turns a bounded window to finish on their own. Only the
// stragglers still active at the deadline are aborted.
beginDrain(reason);
const closePromise = new Promise<Error | null>((resolve) => {
server.close((error) => resolve(error ?? null));
});
const remaining = await waitForTurns(drainMs);
if (remaining > 0) {
logWarn(
`[api] Aborting ${remaining} Fast turn(s) still active after the ${drainMs}ms shutdown drain.`,
);
}
const abortPromise = abortTurns(reason);
const closeError = await closePromise;
await abortPromise;
if (closeError) {
logError('[api] Graceful shutdown failed', closeError);
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading