diff --git a/.fallowrc.json b/.fallowrc.json index ab2831e29..04b432034 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -294,8 +294,9 @@ }, { "comment": "Daemon route handlers are reached only through the dynamic `import()` table in request-handler-chain.ts, which --production analysis cannot follow to a consumer.", - "file": "src/daemon/handlers/{lease,session,snapshot,react-native,record-trace,find,interaction}.ts", + "file": "src/daemon/handlers/{human-control,lease,session,snapshot,react-native,record-trace,find,interaction}.ts", "exports": [ + "handleHumanControlCommand", "handleLeaseCommands", "handleSessionCommands", "handleSnapshotCommands", @@ -307,7 +308,7 @@ }, { "comment": "Dedicated CLI command handlers are reached only through the dynamic `import()` table `dedicatedCliCommandHandlerLoaders` in src/cli/commands/router.ts, which --production analysis cannot follow to a consumer. Same shape as the daemon route-handler entry above; that table is what enumerates this list, so add/remove here whenever a loader is added/removed.", - "file": "src/cli/commands/{auth,connection,daemon,device,proxy,replay,screenshot}.ts", + "file": "src/cli/commands/{auth,connection,daemon,device,proxy,replay,screenshot,takeover}.ts", "exports": [ "authCommand", "connectCommand", @@ -318,7 +319,8 @@ "proxyCommand", "replayCommand", "screenshotCommand", - "diffCommand" + "diffCommand", + "takeoverCommand" ] }, { diff --git a/CONTEXT.md b/CONTEXT.md index 695050ebb..f81904baf 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -75,8 +75,10 @@ Host-global exclusive ownership of one local device by an open session or a sess command. **Device-claim policy**: -A command's declared relationship to local device ownership, including observation, acquisition, -release, and exclusive mutation. +A command's observation, ownership, or exclusive-mutation rule. + +**Human-control hold**: +A device-scoped pause on agent mutations during human operation. ### Commands and routing diff --git a/docs/adr/0007-remote-device-leases.md b/docs/adr/0007-remote-device-leases.md index 6864093c9..4f592267f 100644 --- a/docs/adr/0007-remote-device-leases.md +++ b/docs/adr/0007-remote-device-leases.md @@ -50,3 +50,29 @@ owning lease expiry. Backend-only leases remain valid for older remote clients, while provider-aware clients get device-level contention and clearer recovery. + +## Human control + +Human-control holds coexist with an open remote session. They belong to `LeaseRegistry` and use +the same backend/provider/device contention key as device-aware leases. Hold heartbeat, expiry, +lease preservation, and release refresh are one registry-owned lifecycle. Releasing or expiring the +last hold gives the lease its existing inactivity TTL again; expiry uses the hold's expiry instant. + +Tenant hold operations are ordinary daemon RPCs admitted through `request-admission.ts`. Their +device comes only from the admitted lease. Host administration is a distinct loopback capability +authenticated with the daemon token, never a tenant credential; tenants cannot modify host holds. + +Mutation admission derives from existing recording effects, observation-class inventory, and +observability semantics. Takeover and lease heartbeats are exempt from the mutation fence, not from +their ownership checks. Unknown effects are treated as mutations. A pending activation fences new +mutations and drains those already admitted before reporting active; advisory execution locks alone +do not establish this guarantee for fresh sessions. + +Activation follows the calling RPC or host HTTP request's cancellation signal. A disconnect while +draining removes only that request's pending hold, leaving successor and unrelated holds intact; +canceling activation does not cancel the mutations being drained. Completed holds use their TTL or +explicit release lifecycle. + +Holds, like leases, are in-memory and do not survive daemon restart. Controllers must reconnect and +re-establish them; no persisted hold store is used. Local takeover is deferred: a future host-global +human-control fence must coexist with the local session's device claim, not acquire it exclusively. diff --git a/packages/contracts/src/client-lease.ts b/packages/contracts/src/client-lease.ts index fb27fba66..6f5e6e665 100644 --- a/packages/contracts/src/client-lease.ts +++ b/packages/contracts/src/client-lease.ts @@ -49,3 +49,24 @@ export type CloudArtifactsOptions = AgentDeviceRequestOverrides & { provider?: string; providerSessionId?: string; }; + +export type HumanControlHoldScope = { + backend: LeaseBackend; + leaseProvider?: string; + deviceKey: string; +}; + +export type HumanControlHold = { + id: string; + scope: HumanControlHoldScope; + reason?: string; + state: 'activating' | 'active'; + createdAt: number; + updatedAt: number; + expiresAt?: number; +}; + +export type HumanControlHoldOptions = { + reason?: string; + ttlMs?: number; +}; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index bd9c90a5b..51c964ad0 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -66,6 +66,9 @@ export type { } from '../client-gesture.ts'; export type { CloudArtifactsOptions, + HumanControlHold, + HumanControlHoldOptions, + HumanControlHoldScope, Lease, LeaseAllocateOptions, LeaseOptions, diff --git a/scripts/__tests__/test-file-size-ratchet.test.ts b/scripts/__tests__/test-file-size-ratchet.test.ts index 442b970c7..a4874ba62 100644 --- a/scripts/__tests__/test-file-size-ratchet.test.ts +++ b/scripts/__tests__/test-file-size-ratchet.test.ts @@ -46,7 +46,7 @@ const PINNED_TEST_FILE_LINES: Readonly> = Object.freeze({ 'test/integration/provider-scenarios/android-lifecycle.test.ts': 1556, 'src/utils/__tests__/daemon-client-lifecycle.test.ts': 1413, 'packages/platform-apple/src/runner/__tests__/runner-command-retry.test.ts': 1325, - 'src/__tests__/cli-client-commands.test.ts': 1317, + 'src/__tests__/cli-client-commands.test.ts': 1304, 'src/__tests__/cli-config.test.ts': 1282, 'src/daemon/handlers/__tests__/find.test.ts': 1199, 'packages/platform-apple/src/core/__tests__/perf.test.ts': 1222, diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index 42866b11b..b3382c6ba 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -16,6 +16,7 @@ import type { SettingsUpdateOptions } from '@agent-device/contracts/client'; import { AppError } from '@agent-device/kernel/errors'; import { resolveCliOptions } from '../cli/resolve-cli-options.ts'; import { mkdtempForTestSync } from './test-utils/tmp-dir.ts'; +import { createStubClientLeases } from './test-utils/client-lease-fixtures.ts'; // #1802: the replay client reads the script it names, so CLI-level replay cases need a real file. const REPLAY_SCRIPT_ROOT = mkdtempForTestSync('agent-device-cli-replay-scripts-'); @@ -1214,21 +1215,7 @@ function createStubClient(params: { identifiers: { session: options.session ?? 'default' }, }), }, - leases: { - allocate: async (options) => ({ - leaseId: 'lease-1', - tenantId: options.tenant, - runId: options.runId, - backend: options.leaseBackend ?? 'ios-simulator', - }), - heartbeat: async (options) => ({ - leaseId: options.leaseId, - tenantId: options.tenant ?? 'tenant', - runId: options.runId ?? 'run', - backend: options.leaseBackend ?? 'ios-simulator', - }), - release: async () => ({ released: true }), - }, + leases: createStubClientLeases(), metro: { prepare: params.prepareMetro ?? diff --git a/src/__tests__/daemon-proxy.test.ts b/src/__tests__/daemon-proxy.test.ts index 6d66d9626..6ae2ed85b 100644 --- a/src/__tests__/daemon-proxy.test.ts +++ b/src/__tests__/daemon-proxy.test.ts @@ -210,6 +210,34 @@ test('daemon proxy rejects unauthenticated rpc requests', async (t) => { } }); +test('daemon proxy does not expose local human-control administration', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + + let upstreamCalled = false; + const upstream = http.createServer((_req, res) => { + upstreamCalled = true; + res.end('{}'); + }); + const proxy = createDaemonProxyServer({ + upstreamBaseUrl: `http://127.0.0.1:${await listenOnLoopback(upstream)}`, + upstreamToken: 'daemon-secret', + clientToken: 'proxy-secret', + }); + + try { + const proxyPort = await listenOnLoopback(proxy); + const response = await fetch( + `http://127.0.0.1:${String(proxyPort)}/agent-device/admin/human-control/holds`, + { headers: { authorization: 'Bearer proxy-secret' } }, + ); + assert.equal(response.status, 404); + assert.equal(upstreamCalled, false); + } finally { + await closeLoopbackServer(proxy); + await closeLoopbackServer(upstream); + } +}); + test('daemon proxy leaves health endpoint unauthenticated', async (t) => { if (await skipWhenLoopbackUnavailable(t)) return; diff --git a/src/__tests__/eager-closure-budgets.ts b/src/__tests__/eager-closure-budgets.ts index c5e1e414b..2f7061f35 100644 --- a/src/__tests__/eager-closure-budgets.ts +++ b/src/__tests__/eager-closure-budgets.ts @@ -393,7 +393,7 @@ export const HUB_BUDGETS: Readonly> = Object.freeze({ // readers), `input-audience.ts` (who may write a key), and `common-input-fields.ts` (the table // itself). Every command schema already evaluated all three concerns; the growth is three more // module records for the same code, with no new subtree behind any of them. - 'src/cli.ts': 381, + 'src/cli.ts': 382, 'src/platform-runtime.ts': 47, 'src/core/command-descriptor/registry.ts': 71, 'src/core/command-descriptor/platform-execution-entry.ts': 3, diff --git a/src/__tests__/takeover-command.test.ts b/src/__tests__/takeover-command.test.ts new file mode 100644 index 000000000..b52830a17 --- /dev/null +++ b/src/__tests__/takeover-command.test.ts @@ -0,0 +1,115 @@ +import assert from 'node:assert/strict'; +import { afterEach, beforeEach, test, vi } from 'vitest'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { + renderTakeoverStarted, + renderTakeoverStatus, + takeoverCommand, +} from '../cli/commands/takeover.ts'; +import { + HUMAN_CONTROL_HOLD, + createControlLatch, +} from '../daemon/__tests__/human-control-fixtures.ts'; +import type { DaemonRequest } from '@agent-device/kernel/contracts'; +import { TAKEOVER_CLI_FLAGS } from './test-utils/client-lease-fixtures.ts'; + +const mocks = vi.hoisted(() => ({ writeCommandOutput: vi.fn() })); +vi.mock('../cli/commands/shared.ts', () => ({ writeCommandOutput: mocks.writeCommandOutput })); +beforeEach(() => mocks.writeCommandOutput.mockReset()); +afterEach(() => vi.useRealTimers()); + +function clientForTest() { + const transport = vi.fn(async (req: Omit) => ({ + ok: true as const, + data: + req.positionals?.[0] === 'list' + ? { holds: [HUMAN_CONTROL_HOLD] } + : req.positionals?.[0] === 'remove' + ? { released: true } + : { hold: { ...HUMAN_CONTROL_HOLD, id: req.positionals?.[1] } }, + })); + const client = createAgentDeviceClient( + { session: 'remote-session', leaseId: 'lease-1', tenant: 'tenant-a', runId: 'run-a' }, + { transport }, + ); + return { client, transport }; +} + +test('takeover output explains the hold and release gesture', () => { + assert.match( + renderTakeoverStarted(HUMAN_CONTROL_HOLD), + /Human control active for ios:mobile:sim-1/, + ); + assert.match(renderTakeoverStarted(HUMAN_CONTROL_HOLD), /Press Ctrl\+C/); + assert.equal(renderTakeoverStatus([]), 'No active human-control holds.'); + assert.match(renderTakeoverStatus([HUMAN_CONTROL_HOLD]), /operator-1: ios:mobile:sim-1/); +}); + +test('takeover status and release use the configured lease client', async () => { + const { client, transport } = clientForTest(); + await takeoverCommand({ client, flags: TAKEOVER_CLI_FLAGS, positionals: ['status'] }); + await takeoverCommand({ + client, + flags: TAKEOVER_CLI_FLAGS, + positionals: ['release', 'operator-1'], + }); + assert.deepEqual( + transport.mock.calls.map(([req]) => req.positionals), + [['list'], ['remove', 'operator-1']], + ); + for (const [req] of transport.mock.calls) { + assert.equal(req.command, 'human_control'); + assert.equal(req.session, 'remote-session'); + assert.equal(req.meta?.leaseId, 'lease-1'); + } +}); + +test('takeover rejects malformed actions without contacting the daemon', async () => { + const { client, transport } = clientForTest(); + for (const positionals of [['release'], ['status', 'extra'], ['unknown']]) { + await assert.rejects(takeoverCommand({ client, flags: TAKEOVER_CLI_FLAGS, positionals }), { + code: 'INVALID_ARGS', + }); + } + assert.equal(transport.mock.calls.length, 0); +}); + +test('foreground takeover renews its admitted lease hold and releases it on Ctrl+C', async () => { + vi.useFakeTimers(); + const { client, transport } = clientForTest(); + const started = createControlLatch(); + mocks.writeCommandOutput.mockImplementationOnce(() => started.resolve()); + const pending = takeoverCommand({ client, flags: TAKEOVER_CLI_FLAGS, positionals: [] }); + await started.promise; + await vi.advanceTimersByTimeAsync(5_000); + process.emit('SIGINT'); + assert.equal(await pending, true); + assert.deepEqual( + transport.mock.calls.map(([req]) => req.positionals?.[0]), + ['put', 'put', 'remove'], + ); + assert.equal( + transport.mock.calls[0]?.[0].positionals?.[1], + transport.mock.calls[2]?.[0].positionals?.[1], + ); + const input = JSON.parse(transport.mock.calls[0]?.[0].positionals?.[2] ?? '{}') as Record< + string, + unknown + >; + assert.equal(input.ttlMs, 15_000); + assert.equal(input.scope, undefined); +}); + +test('a failed heartbeat stops foreground takeover and attempts release', async () => { + vi.useFakeTimers(); + const { client, transport } = clientForTest(); + const started = createControlLatch(); + mocks.writeCommandOutput.mockImplementationOnce(() => started.resolve()); + const pending = takeoverCommand({ client, flags: TAKEOVER_CLI_FLAGS, positionals: [] }); + const rejected = assert.rejects(pending, /heartbeat failed/); + await started.promise; + transport.mockRejectedValueOnce(new Error('heartbeat failed')); + await vi.advanceTimersByTimeAsync(5_000); + await rejected; + assert.equal(transport.mock.calls.at(-1)?.[0].positionals?.[0], 'remove'); +}); diff --git a/src/__tests__/test-utils/client-lease-fixtures.ts b/src/__tests__/test-utils/client-lease-fixtures.ts new file mode 100644 index 000000000..0b48559d6 --- /dev/null +++ b/src/__tests__/test-utils/client-lease-fixtures.ts @@ -0,0 +1,29 @@ +import type { AgentDeviceClient } from '../../agent-device-client.ts'; +import type { CliFlags } from '@agent-device/contracts/command'; + +export const TAKEOVER_CLI_FLAGS: CliFlags = { json: true, help: false, version: false }; + +export function createStubClientLeases(): AgentDeviceClient['leases'] { + return { + allocate: async (options) => ({ + leaseId: 'lease-1', + tenantId: options.tenant, + runId: options.runId, + backend: options.leaseBackend ?? 'ios-simulator', + }), + heartbeat: async (options) => ({ + leaseId: options.leaseId, + tenantId: options.tenant ?? 'tenant', + runId: options.runId ?? 'run', + backend: options.leaseBackend ?? 'ios-simulator', + }), + release: async () => ({ released: true }), + humanControl: { + list: async () => [], + put: async () => { + throw new Error('Unexpected takeover in this fixture'); + }, + remove: async () => false, + }, + }; +} diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 4dada8f5f..fa3d52152 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -443,6 +443,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { 'trigger-app-event': GENERIC_REPLAY_LINE, 'tv-remote': GENERIC_REPLAY_LINE, viewport: GENERIC_REPLAY_LINE, + human_control: { waived: 'host-local control commands are never recorded in replay scripts' }, install_source: GENERIC_REPLAY_LINE, lease_allocate: GENERIC_REPLAY_LINE, lease_heartbeat: GENERIC_REPLAY_LINE, diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index 52fe66573..a06d10418 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -19,7 +19,6 @@ import type { DragOptions, FlingOptions, InternalRequestOptions, - Lease, MaterializationReleaseOptions, PanOptions, PinchOptions, @@ -79,6 +78,7 @@ import { type MetroSessionHints, } from './metro/metro-session-hints.ts'; import { isRecord } from '@agent-device/kernel/record'; +import { createLeaseClient } from './client/lease-client.ts'; import { readScreenshotResultData } from './utils/screenshot-result.ts'; type ProjectedSystemCommandClient = ProjectedNavigationCommandClient & @@ -316,21 +316,7 @@ export function createAgentDeviceClient( }), ), }, - leases: { - allocate: async (options) => - normalizeLease( - await execute(INTERNAL_COMMANDS.leaseAllocate, [], { - ...options, - leaseId: undefined, - }), - ), - heartbeat: async (options) => - normalizeLease(await execute(INTERNAL_COMMANDS.leaseHeartbeat, [], options)), - release: async (options) => { - const data = await execute(INTERNAL_COMMANDS.leaseRelease, [], options); - return { released: data.released === true, provider: readObject(data.provider) }; - }, - }, + leases: createLeaseClient(execute), metro: { prepare: async (options: MetroPrepareOptions) => { const result = await prepareMetroRuntime({ @@ -677,23 +663,4 @@ function clearMetroSessionHintsQuietly( } } -function normalizeLease(data: Record): Lease { - const rawLease = data.lease; - if (!isRecord(rawLease)) { - throw new Error('Invalid lease response from daemon'); - } - return { - leaseId: readRequiredString(rawLease, 'leaseId'), - tenantId: readRequiredString(rawLease, 'tenantId'), - runId: readRequiredString(rawLease, 'runId'), - backend: readRequiredString(rawLease, 'backend') as Lease['backend'], - leaseProvider: readOptionalString(rawLease, 'leaseProvider'), - clientId: readOptionalString(rawLease, 'clientId'), - deviceKey: readOptionalString(rawLease, 'deviceKey'), - createdAt: typeof rawLease.createdAt === 'number' ? rawLease.createdAt : undefined, - heartbeatAt: typeof rawLease.heartbeatAt === 'number' ? rawLease.heartbeatAt : undefined, - expiresAt: typeof rawLease.expiresAt === 'number' ? rawLease.expiresAt : undefined, - }; -} - export type * from './client/client-types.ts'; diff --git a/src/cli-schema/cli-help-command-usage.test.ts b/src/cli-schema/cli-help-command-usage.test.ts index eb504a483..1e13d67c8 100644 --- a/src/cli-schema/cli-help-command-usage.test.ts +++ b/src/cli-schema/cli-help-command-usage.test.ts @@ -174,6 +174,17 @@ test('proxy command help describes tunnel usage', async () => { assert.doesNotMatch(help, /agent-device-proxy/); }); +test('takeover command help documents lease-bound foreground and host API flows', async () => { + const help = await usageForCommand('takeover'); + if (help === null) throw new Error('Expected command help text'); + assert.match(help, /Usage:\s+agent-device takeover/); + assert.match(help, /foreground command pauses state-changing agent commands/); + assert.match(help, /until Ctrl\+C/); + assert.match(help, /\/admin\/human-control\/holds/); + assert.match(help, /active remote lease device/); + assert.match(help, /do not survive daemon restart/); +}); + test('connect command help lists lease id in usage and flags', async () => { const help = await usageForCommand('connect'); if (help === null) throw new Error('Expected command help text'); diff --git a/src/cli-schema/cli-help-topics.test.ts b/src/cli-schema/cli-help-topics.test.ts index c59c6fb4e..86f51af71 100644 --- a/src/cli-schema/cli-help-topics.test.ts +++ b/src/cli-schema/cli-help-topics.test.ts @@ -167,6 +167,14 @@ test('usageForCommand resolves Maestro compatibility help topic', async () => { assert.doesNotMatch(help, /issues\/558/); }); +test('remote help documents lease-bound takeover and distinct host administration', async () => { + const help = await usageForCommand('remote'); + if (help === null) throw new Error('Expected remote help text'); + assert.match(help, /agent-device takeover --session remote-session/); + assert.match(help, /authenticated GET\/PUT\/DELETE requests/); + assert.match(help, /not forwarded by agent-device proxy/); +}); + test('usageForCommand resolves workflow help topic', async () => { const help = await usageForCommand('workflow'); if (help === null) throw new Error('Expected workflow help text'); diff --git a/src/cli-schema/cli-help.ts b/src/cli-schema/cli-help.ts index 6a4b4a11a..9d7754a38 100644 --- a/src/cli-schema/cli-help.ts +++ b/src/cli-schema/cli-help.ts @@ -758,6 +758,13 @@ Direct proxy flow for a remote Mac/simulator: agent-device close agent-device disconnect +Human takeover of a leased remote device: + Run agent-device takeover using the active remote connection and session. It pauses state-changing agent commands until Ctrl+C while snapshots and other read-only diagnostics remain available. Tenant requests can control only their admitted lease device. Local takeover without a remote device lease is not supported. + agent-device takeover --session remote-session + agent-device takeover status + agent-device takeover release + An HTTP-mode daemon also accepts authenticated GET/PUT/DELETE requests at /admin/human-control/holds on its loopback listener. Host administrators supply the exact lease backend/provider/device key and use the local daemon token, not a tenant credential. This host-admin route is intentionally not forwarded by agent-device proxy. Holds do not survive daemon restart; re-establish them after reconnecting. + Cloud profile flow: agent-device connect agent-device open com.example.app diff --git a/src/cli-schema/command-overrides.ts b/src/cli-schema/command-overrides.ts index 29181b5cb..8f10b4da7 100644 --- a/src/cli-schema/command-overrides.ts +++ b/src/cli-schema/command-overrides.ts @@ -142,6 +142,29 @@ const SCHEMA_ONLY_CLI_COMMAND_SCHEMAS = { listUsageOverride: 'proxy', allowedFlags: ['proxyHost', 'proxyPort', 'daemonAuthToken', 'stateDir'], }, + takeover: { + text: { + summary: 'Pause agent interactions while a person controls a device', + description: + 'Temporarily hand control of the active remote lease device to a person. The foreground command pauses state-changing agent commands, renews the hold until Ctrl+C, and then releases it. Read-only diagnostics remain available. Uses the active connection and --session, with normal tenant and lease admission. status lists holds on that device; release removes a hold owned by the admitted lease. The host-only /admin/human-control/holds API uses the separate local daemon token. Holds do not survive daemon restart. Local takeover without a remote device lease is not supported.', + }, + usageOverride: 'takeover [status | release ] [--session ]', + listUsageOverride: 'takeover [status|release]', + positionalArgs: ['status|release?', 'hold-id?'], + supportedFlags: [ + 'stateDir', + 'session', + 'remoteConfig', + 'daemonBaseUrl', + 'daemonAuthToken', + 'daemonTransport', + 'tenant', + 'runId', + 'leaseId', + 'leaseBackend', + 'sessionIsolation', + ], + }, 'react-devtools': { text: { summary: 'Inspect components, hooks, and render profiles', diff --git a/src/cli.ts b/src/cli.ts index c0c96773b..62ce54563 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -91,6 +91,7 @@ const REMOTE_MATERIALIZATION_DEFERRED_COMMANDS = new Set([ 'metro', 'proxy', 'session', + 'takeover', ]); export async function runCli(argv: string[], deps: CliDeps = DEFAULT_CLI_DEPS): Promise { diff --git a/src/cli/commands/router.ts b/src/cli/commands/router.ts index 1d8408e0a..da787a1ab 100644 --- a/src/cli/commands/router.ts +++ b/src/cli/commands/router.ts @@ -16,6 +16,7 @@ const dedicatedCliCommandHandlerLoaders = { daemon: async () => (await import('./daemon.ts')).daemonCommand, device: async () => (await import('./device.ts')).deviceCommand, proxy: async () => (await import('./proxy.ts')).proxyCommand, + takeover: async () => (await import('./takeover.ts')).takeoverCommand, replay: async () => (await import('./replay.ts')).replayCommand, screenshot: async () => (await import('./screenshot.ts')).screenshotCommand, diff: async () => (await import('./screenshot.ts')).diffCommand, diff --git a/src/cli/commands/takeover.ts b/src/cli/commands/takeover.ts new file mode 100644 index 000000000..b5743742a --- /dev/null +++ b/src/cli/commands/takeover.ts @@ -0,0 +1,125 @@ +import { randomUUID } from 'node:crypto'; +import type { CliFlags } from '@agent-device/contracts/command'; +import { AppError } from '@agent-device/kernel/errors'; +import type { AgentDeviceClient } from '../../agent-device-client.ts'; +import type { HumanControlHold } from '@agent-device/contracts/client'; +import { writeCommandOutput } from './shared.ts'; +import type { ClientCommandHandler } from './router-types.ts'; + +const FOREGROUND_HOLD_TTL_MS = 15_000; +const FOREGROUND_HEARTBEAT_MS = 5_000; +export const takeoverCommand: ClientCommandHandler = async ({ positionals, flags, client }) => { + const action = positionals[0]?.toLowerCase(); + if (action === 'status') { + if (positionals.length !== 1) { + throw new AppError('INVALID_ARGS', 'takeover status does not accept additional arguments.'); + } + await showTakeoverStatus(flags, client); + return true; + } + if (action === 'release') { + if (positionals.length !== 2 || !positionals[1]) { + throw new AppError('INVALID_ARGS', 'takeover release requires a hold id.'); + } + await releaseTakeover(flags, client, positionals[1]); + return true; + } + if (positionals.length > 0) { + throw new AppError('INVALID_ARGS', 'takeover accepts only: status or release .'); + } + + await runForegroundTakeover(flags, client); + return true; +}; + +async function runForegroundTakeover( + flags: CliFlags, + agentDeviceClient: AgentDeviceClient, +): Promise { + const holdId = `takeover-${randomUUID()}`; + const input = { + reason: 'Human is interacting with the simulator or device.', + ttlMs: FOREGROUND_HOLD_TTL_MS, + }; + const client = agentDeviceClient.leases.humanControl; + const hold = await client.put(holdId, input); + writeCommandOutput(flags, { hold, state: 'active' }, () => renderTakeoverStarted(hold)); + + let heartbeatError: unknown; + let heartbeatRequest: Promise | undefined; + let released = false; + let finish: (() => void) | undefined; + const stopped = new Promise((resolve) => { + finish = resolve; + }); + const stop = () => finish?.(); + process.once('SIGINT', stop); + process.once('SIGTERM', stop); + const heartbeat = setInterval(() => { + if (heartbeatRequest) return; + heartbeatRequest = client + .put(holdId, input) + .then(() => undefined) + .catch((error: unknown) => { + heartbeatError = error; + finish?.(); + }) + .finally(() => { + heartbeatRequest = undefined; + }); + }, FOREGROUND_HEARTBEAT_MS); + + try { + await stopped; + } finally { + clearInterval(heartbeat); + process.off('SIGINT', stop); + process.off('SIGTERM', stop); + await heartbeatRequest; + released = await client.remove(holdId).catch(() => false); + } + if (heartbeatError) throw heartbeatError; + if (!flags.json) { + process.stdout.write( + released + ? 'Human control released. Agent interactions are enabled.\n' + : 'Release could not be confirmed. The safety TTL will re-enable agent interactions automatically.\n', + ); + } +} + +async function showTakeoverStatus(flags: CliFlags, client: AgentDeviceClient): Promise { + const holds = await client.leases.humanControl.list(); + writeCommandOutput(flags, { holds }, () => renderTakeoverStatus(holds)); +} + +async function releaseTakeover( + flags: CliFlags, + client: AgentDeviceClient, + holdId: string, +): Promise { + const released = await client.leases.humanControl.remove(holdId); + writeCommandOutput(flags, { holdId, released }, () => + released ? `Released human-control hold ${holdId}.` : `No active hold found for ${holdId}.`, + ); +} + +export function renderTakeoverStarted(hold: HumanControlHold): string { + const target = hold.scope.deviceKey; + return [ + `Human control active for ${target}.`, + 'Agent interactions are paused. Press Ctrl+C to return control.', + `Hold: ${hold.id}`, + ].join('\n'); +} + +export function renderTakeoverStatus(holds: HumanControlHold[]): string { + if (holds.length === 0) return 'No active human-control holds.'; + return [ + 'Active human-control holds:', + ...holds.map((hold) => { + const target = hold.scope.deviceKey; + return ` ${hold.id}: ${target}`; + }), + ].join('\n'); +} diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 772be788e..5975ea921 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -98,6 +98,8 @@ import type { Lease, LeaseAllocateOptions, LeaseScopedOptions, + HumanControlHold, + HumanControlHoldOptions, LogsOptions, LongPressOptions, MaterializationReleaseOptions, @@ -239,6 +241,15 @@ export type AgentDeviceClient = { release: ( options: LeaseScopedOptions, ) => Promise<{ released: boolean; provider?: CloudProviderSessionResult }>; + humanControl: { + list: (options?: AgentDeviceRequestOverrides) => Promise; + put: ( + id: string, + input?: HumanControlHoldOptions, + options?: AgentDeviceRequestOverrides, + ) => Promise; + remove: (id: string, options?: AgentDeviceRequestOverrides) => Promise; + }; }; metro: { prepare: (options: MetroPrepareOptions) => Promise; diff --git a/src/client/lease-client.test.ts b/src/client/lease-client.test.ts new file mode 100644 index 000000000..cfcf9dfcb --- /dev/null +++ b/src/client/lease-client.test.ts @@ -0,0 +1,75 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { AppError } from '@agent-device/kernel/errors'; +import { createAgentDeviceClient } from '../agent-device-client.ts'; +import { + HUMAN_CONTROL_HOLD, + HUMAN_CONTROL_SCOPE, +} from '../daemon/__tests__/human-control-fixtures.ts'; +import type { DaemonRequest } from '@agent-device/kernel/contracts'; + +test('human-control client carries normal remote authentication and full lease metadata', async () => { + const requests: Array> = []; + const client = createAgentDeviceClient( + { + daemonBaseUrl: 'https://daemon.example.test', + daemonAuthToken: 'tenant-token', + tenant: 'tenant-a', + runId: 'run-a', + leaseId: 'lease-1', + clientId: 'client-a', + leaseBackend: HUMAN_CONTROL_SCOPE.backend, + leaseProvider: HUMAN_CONTROL_SCOPE.leaseProvider, + deviceKey: HUMAN_CONTROL_SCOPE.deviceKey, + }, + { + transport: async (request, context) => { + assert.equal(context?.authToken, 'tenant-token'); + requests.push(request); + return { + ok: true, + data: { hold: HUMAN_CONTROL_HOLD, holds: [HUMAN_CONTROL_HOLD], released: true }, + }; + }, + }, + ); + assert.equal( + (await client.leases.humanControl.put('operator-1', { ttlMs: 15_000 })).id, + 'operator-1', + ); + assert.equal((await client.leases.humanControl.list()).length, 1); + assert.equal(await client.leases.humanControl.remove('operator-1'), true); + for (const request of requests) { + assert.equal(request.command, 'human_control'); + assert.equal(request.meta?.leaseId, 'lease-1'); + assert.equal(request.meta?.leaseProvider, 'proxy'); + assert.equal(request.meta?.deviceKey, HUMAN_CONTROL_SCOPE.deviceKey); + assert.equal(request.meta?.clientId, 'client-a'); + } +}); + +test('lease client rejects invalid control responses and preserves normalized errors', async () => { + const client = createAgentDeviceClient({}, { transport: async () => ({ ok: true, data: {} }) }); + await assert.rejects(client.leases.humanControl.put('operator-1'), { code: 'COMMAND_FAILED' }); + await assert.rejects(client.leases.humanControl.list(), { code: 'COMMAND_FAILED' }); + const denied = createAgentDeviceClient( + {}, + { + transport: async () => ({ + ok: false, + error: { + code: 'UNAUTHORIZED', + message: 'Wrong lease', + details: { reason: 'LEASE_SCOPE_MISMATCH' }, + }, + }), + }, + ); + await assert.rejects( + denied.leases.humanControl.remove('operator-1'), + (error: unknown) => + error instanceof AppError && + error.code === 'UNAUTHORIZED' && + error.details?.reason === 'LEASE_SCOPE_MISMATCH', + ); +}); diff --git a/src/client/lease-client.ts b/src/client/lease-client.ts new file mode 100644 index 000000000..eb2eac8e1 --- /dev/null +++ b/src/client/lease-client.ts @@ -0,0 +1,93 @@ +import type { + HumanControlHold, + InternalRequestOptions, + Lease, +} from '@agent-device/contracts/client'; +import { AppError } from '@agent-device/kernel/errors'; +import { INTERNAL_COMMANDS } from '../command-catalog.ts'; +import { isRecord } from '@agent-device/kernel/record'; +import type { AgentDeviceClient } from './client-types.ts'; +import { readOptionalString, readRequiredString } from './client-normalizers.ts'; + +export function createLeaseClient( + execute: ( + command: string, + positionals?: string[], + options?: InternalRequestOptions, + ) => Promise>, +): AgentDeviceClient['leases'] { + const control = async (positionals: string[], options?: InternalRequestOptions) => + await execute(INTERNAL_COMMANDS.humanControl, positionals, options); + return { + allocate: async (options) => + normalizeLease( + await execute(INTERNAL_COMMANDS.leaseAllocate, [], { ...options, leaseId: undefined }), + ), + heartbeat: async (options) => + normalizeLease(await execute(INTERNAL_COMMANDS.leaseHeartbeat, [], options)), + release: async (options) => { + const data = await execute(INTERNAL_COMMANDS.leaseRelease, [], options); + return { + released: data.released === true, + provider: isRecord(data.provider) ? data.provider : undefined, + }; + }, + humanControl: { + list: async (options) => { + const data = await control(['list'], options); + if (!Array.isArray(data.holds)) + throw new AppError('COMMAND_FAILED', 'Daemon did not return human-control holds.'); + return data.holds.map(normalizeHumanControlHold); + }, + put: async (id, input = {}, options) => { + const data = await control(['put', id, JSON.stringify(input)], options); + return normalizeHumanControlHold(data.hold); + }, + remove: async (id, options) => (await control(['remove', id], options)).released === true, + }, + }; +} + +function normalizeHumanControlHold(value: unknown): HumanControlHold { + if ( + !isRecord(value) || + !isRecord(value.scope) || + (value.state !== 'active' && value.state !== 'activating') || + typeof value.createdAt !== 'number' || + typeof value.updatedAt !== 'number' + ) { + throw new AppError('COMMAND_FAILED', 'Daemon returned an invalid human-control hold.'); + } + return { + id: readRequiredString(value, 'id'), + scope: { + backend: readRequiredString(value.scope, 'backend') as HumanControlHold['scope']['backend'], + leaseProvider: readOptionalString(value.scope, 'leaseProvider'), + deviceKey: readRequiredString(value.scope, 'deviceKey'), + }, + state: value.state, + reason: readOptionalString(value, 'reason'), + createdAt: value.createdAt, + updatedAt: value.updatedAt, + ...(typeof value.expiresAt === 'number' ? { expiresAt: value.expiresAt } : {}), + }; +} + +function normalizeLease(data: Record): Lease { + const rawLease = data.lease; + if (!isRecord(rawLease)) { + throw new Error('Invalid lease response from daemon'); + } + return { + leaseId: readRequiredString(rawLease, 'leaseId'), + tenantId: readRequiredString(rawLease, 'tenantId'), + runId: readRequiredString(rawLease, 'runId'), + backend: readRequiredString(rawLease, 'backend') as Lease['backend'], + leaseProvider: readOptionalString(rawLease, 'leaseProvider'), + clientId: readOptionalString(rawLease, 'clientId'), + deviceKey: readOptionalString(rawLease, 'deviceKey'), + createdAt: typeof rawLease.createdAt === 'number' ? rawLease.createdAt : undefined, + heartbeatAt: typeof rawLease.heartbeatAt === 'number' ? rawLease.heartbeatAt : undefined, + expiresAt: typeof rawLease.expiresAt === 'number' ? rawLease.expiresAt : undefined, + }; +} diff --git a/src/core/command-descriptor/__tests__/device-claim-policy.test.ts b/src/core/command-descriptor/__tests__/device-claim-policy.test.ts index 2cdeaf362..74ed32391 100644 --- a/src/core/command-descriptor/__tests__/device-claim-policy.test.ts +++ b/src/core/command-descriptor/__tests__/device-claim-policy.test.ts @@ -50,7 +50,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set 'reinstall', 'shutdown', ], - observe: ['apps', 'appstate', 'capabilities', 'device', 'devices', 'doctor'], + observe: ['apps', 'appstate', 'capabilities', 'device', 'devices', 'doctor', 'takeover'], none: [ 'artifacts', 'auth', @@ -61,6 +61,7 @@ test('every command that deviates from require-owner is a reviewed, diffable set 'daemon', 'debug', 'disconnect', + 'human_control', 'install-from-source', 'lease_allocate', 'lease_heartbeat', diff --git a/src/core/command-descriptor/__tests__/parity.test.ts b/src/core/command-descriptor/__tests__/parity.test.ts index 9c2e5b1e7..71735fa54 100644 --- a/src/core/command-descriptor/__tests__/parity.test.ts +++ b/src/core/command-descriptor/__tests__/parity.test.ts @@ -253,6 +253,18 @@ test('recordsSessionAction is explicit on every raw descriptor and drives daemon }); test('recordingEffect resolves request-sensitive observation and mutation subcommands', () => { + assert.equal( + resolveCommandRecordingEffect({ command: 'clipboard', positionals: ['read'], flags: {} }), + 'observes-app', + ); + assert.equal( + resolveCommandRecordingEffect({ + command: 'clipboard', + positionals: ['write', 'text'], + flags: {}, + }), + 'mutates-app', + ); assert.equal( resolveCommandRecordingEffect({ command: 'keyboard', positionals: ['status'], flags: {} }), 'observes-app', diff --git a/src/core/command-descriptor/daemon-command-descriptor.ts b/src/core/command-descriptor/daemon-command-descriptor.ts index 713fb39ff..2fa89e82c 100644 --- a/src/core/command-descriptor/daemon-command-descriptor.ts +++ b/src/core/command-descriptor/daemon-command-descriptor.ts @@ -8,6 +8,7 @@ export type SessionCommandKind = 'inventory' | 'state' | 'observability' | 'publ * `request-handler-chain.ts` must cover every member (`satisfies Record<…>`). */ export type DaemonCommandRoute = + | 'humanControl' | 'lease' | 'session' | 'snapshot' diff --git a/src/core/command-descriptor/registry.ts b/src/core/command-descriptor/registry.ts index c86dd28b0..84f342e23 100644 --- a/src/core/command-descriptor/registry.ts +++ b/src/core/command-descriptor/registry.ts @@ -214,6 +214,9 @@ const findRecordingEffect = (req: DispatchedCommand): RecordingEffect => { } }; +const clipboardRecordingEffect = (req: DispatchedCommand): RecordingEffect => + readOnlySubactionRecordingEffect(req, new Set(['read']), ''); + function readOnlySubactionRefFrameEffect( req: DispatchedCommand, readOnlyActions: ReadonlySet, @@ -400,7 +403,37 @@ function postActionObservation(command: string): PostActionObservationSupport { const ownerFilesEnabled = typeof __OWNER_FILES__ === 'undefined' || __OWNER_FILES__; +const DEPLOY_APP_COMMAND_DESCRIPTOR = { + deviceClaimPolicy: 'transient-exclusive', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/management/install.ts'] as const } : {}), + catalog: { group: 'public' }, + frameworkTier: 'extended', + recordsSessionAction: true, + recordingEffect: 'mutates-app', + daemon: { route: 'session', refFrameEffect: 'may-invalidate' }, + platformExecution: { kind: 'device-runtime', use: deployAppUse }, + timeoutPolicy: INSTALL_TIMEOUT_POLICY, + batchable: true, +} as const; + export const RAW_COMMAND_DESCRIPTORS = [ + { + name: 'human_control', + deviceClaimPolicy: 'none', + ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/human-control.ts'] as const } : {}), + catalog: { group: 'internal', key: 'humanControl' }, + recordsSessionAction: false, + daemon: { + route: 'humanControl', + refFrameEffect: 'preserve', + selectorValidationExempt: true, + skipSessionlessProviderDevice: allowAnyDeviceSessionless, + }, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: false, + platformExecution: NO_PLATFORM_EXECUTION, + }, + // -- lease (route: lease) -- { name: 'lease_allocate', @@ -408,7 +441,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/lease.ts'] as const } : {}), catalog: { group: 'internal', key: 'leaseAllocate' }, recordsSessionAction: false, - daemon: { route: 'lease', refFrameEffect: 'preserve', ...ADMISSION_AND_LOCK_EXEMPT }, + daemon: { + route: 'lease', + refFrameEffect: 'preserve', + ...ADMISSION_AND_LOCK_EXEMPT, + }, timeoutPolicy: LEASE_ALLOCATE_TIMEOUT_POLICY, batchable: false, platformExecution: NO_PLATFORM_EXECUTION, @@ -419,7 +456,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/lease.ts'] as const } : {}), catalog: { group: 'internal', key: 'leaseHeartbeat' }, recordsSessionAction: false, - daemon: { route: 'lease', refFrameEffect: 'preserve', ...ADMISSION_AND_LOCK_EXEMPT }, + daemon: { + route: 'lease', + refFrameEffect: 'preserve', + ...ADMISSION_AND_LOCK_EXEMPT, + }, timeoutPolicy: LEASE_TIMEOUT_POLICY, batchable: false, platformExecution: NO_PLATFORM_EXECUTION, @@ -430,7 +471,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ ...(ownerFilesEnabled ? { ownerFiles: ['src/daemon/handlers/lease.ts'] as const } : {}), catalog: { group: 'internal', key: 'leaseRelease' }, recordsSessionAction: false, - daemon: { route: 'lease', refFrameEffect: 'preserve', ...ADMISSION_AND_LOCK_EXEMPT }, + daemon: { + route: 'lease', + refFrameEffect: 'preserve', + ...ADMISSION_AND_LOCK_EXEMPT, + }, timeoutPolicy: LEASE_TIMEOUT_POLICY, batchable: false, platformExecution: NO_PLATFORM_EXECUTION, @@ -442,7 +487,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'lease', refFrameEffect: 'preserve', ...ADMISSION_AND_LOCK_EXEMPT }, + daemon: { + route: 'lease', + refFrameEffect: 'preserve', + ...ADMISSION_AND_LOCK_EXEMPT, + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: false, platformExecution: NO_PLATFORM_EXECUTION, @@ -570,7 +619,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'may-invalidate', sessionKind: 'state' }, + daemon: { + route: 'session', + refFrameEffect: 'may-invalidate', + sessionKind: 'state', + }, platformExecution: { kind: 'device-runtime', uses: deviceBootRuntimeUses }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, @@ -582,7 +635,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'may-invalidate', sessionKind: 'state' }, + daemon: { + route: 'session', + refFrameEffect: 'may-invalidate', + sessionKind: 'state', + }, platformExecution: { kind: 'device-runtime', use: shutdownTargetUse }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, @@ -594,7 +651,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public', key: 'appState' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'preserve', sessionKind: 'state' }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + sessionKind: 'state', + }, platformExecution: { kind: 'device-runtime', uses: appStateRuntimeUses }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, @@ -607,7 +668,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ frameworkTier: 'extended', recordsSessionAction: true, recordingEffect: 'observes-app', - daemon: { route: 'session', refFrameEffect: 'preserve', sessionKind: 'observability' }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + sessionKind: 'observability', + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: perfRuntimePlanUses }, @@ -619,7 +684,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'preserve', sessionKind: 'observability' }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + sessionKind: 'observability', + }, platformExecution: { kind: 'device-runtime', uses: appLogRuntimePlanUses }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, @@ -651,7 +720,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'preserve', sessionKind: 'observability' }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + sessionKind: 'observability', + }, platformExecution: { kind: 'device-runtime', use: networkDumpUse }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, @@ -663,7 +736,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ catalog: { group: 'public' }, frameworkTier: 'extended', recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'preserve', sessionKind: 'observability' }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + sessionKind: 'observability', + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: audioRuntimePlanUses }, @@ -742,8 +819,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ // whichever action-selected fact (`readClipboard`/`writeClipboard`) the parsed subcommand // names, and the only execution is that one bound operation (ADR 0019 §9). recordsSessionAction: true, - recordingEffect: 'observes-app', - daemon: { route: 'session', refFrameEffect: 'preserve' }, + recordingEffect: clipboardRecordingEffect, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: clipboardRuntimePlanUses }, @@ -770,29 +850,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ }, { name: 'install', - deviceClaimPolicy: 'transient-exclusive', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/management/install.ts'] as const } : {}), - catalog: { group: 'public' }, - frameworkTier: 'extended', - recordsSessionAction: true, - recordingEffect: 'mutates-app', - daemon: { route: 'session', refFrameEffect: 'may-invalidate' }, - platformExecution: { kind: 'device-runtime', use: deployAppUse }, - timeoutPolicy: INSTALL_TIMEOUT_POLICY, - batchable: true, + ...DEPLOY_APP_COMMAND_DESCRIPTOR, }, { name: 'reinstall', - deviceClaimPolicy: 'transient-exclusive', - ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/management/install.ts'] as const } : {}), - catalog: { group: 'public' }, - frameworkTier: 'extended', - recordsSessionAction: true, - recordingEffect: 'mutates-app', - daemon: { route: 'session', refFrameEffect: 'may-invalidate' }, - platformExecution: { kind: 'device-runtime', use: deployAppUse }, - timeoutPolicy: INSTALL_TIMEOUT_POLICY, - batchable: true, + ...DEPLOY_APP_COMMAND_DESCRIPTOR, }, { name: 'install_source', @@ -816,7 +878,11 @@ export const RAW_COMMAND_DESCRIPTORS = [ : {}), catalog: { group: 'internal', key: 'releaseMaterializedPaths' }, recordsSessionAction: false, - daemon: { route: 'session', refFrameEffect: 'preserve', ...REQUEST_EXECUTION_EXEMPT }, + daemon: { + route: 'session', + refFrameEffect: 'preserve', + ...REQUEST_EXECUTION_EXEMPT, + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: false, platformExecution: NO_PLATFORM_EXECUTION, @@ -996,7 +1062,10 @@ export const RAW_COMMAND_DESCRIPTORS = [ // how long a transient sheet takes to appear is family mechanics, not request policy. recordsSessionAction: true, recordingEffect: alertRecordingEffect, - daemon: { route: 'snapshot', refFrameEffect: alertRefFrameEffect }, + daemon: { + route: 'snapshot', + refFrameEffect: alertRefFrameEffect, + }, timeoutPolicy: DEFAULT_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: alertRuntimePlanUses }, @@ -1075,7 +1144,10 @@ export const RAW_COMMAND_DESCRIPTORS = [ frameworkTier: 'core', recordsSessionAction: true, recordingEffect: findRecordingEffect, - daemon: { route: 'find', refFrameEffect: 'may-invalidate' }, + daemon: { + route: 'find', + refFrameEffect: 'may-invalidate', + }, timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, batchable: true, platformExecution: { kind: 'device-runtime', uses: findRuntimePlanUses }, @@ -1521,6 +1593,17 @@ export const RAW_COMMAND_DESCRIPTORS = [ mcpExposed: false, platformExecution: NO_PLATFORM_EXECUTION, }, + { + name: 'takeover', + deviceClaimPolicy: 'observe', + ...(ownerFilesEnabled ? { ownerFiles: ['src/cli/commands/takeover.ts'] as const } : {}), + catalog: { group: 'local-cli' }, + recordsSessionAction: false, + timeoutPolicy: DEFAULT_TIMEOUT_POLICY, + batchable: false, + mcpExposed: false, + platformExecution: { kind: 'inventory', use: inventoryUse }, + }, { name: 'react-devtools', deviceClaimPolicy: 'none', diff --git a/src/daemon/__tests__/daemon-command-registry.test.ts b/src/daemon/__tests__/daemon-command-registry.test.ts index dce44a493..2446ba6ea 100644 --- a/src/daemon/__tests__/daemon-command-registry.test.ts +++ b/src/daemon/__tests__/daemon-command-registry.test.ts @@ -6,6 +6,7 @@ import { canRunReplayScopedAction, getDaemonCommandRoute, getSessionCommandKind, + isHumanControlMutation, isLeaseAdmissionExempt, shouldBlockForInvalidRecording, shouldGuardAndroidBlockingDialog, @@ -18,6 +19,7 @@ import { import type { DaemonRequest } from '../types.ts'; test('daemon command registry owns specialized handler routes', () => { + assert.equal(getDaemonCommandRoute(INTERNAL_COMMANDS.humanControl), 'humanControl'); for (const command of [ INTERNAL_COMMANDS.leaseAllocate, INTERNAL_COMMANDS.leaseHeartbeat, @@ -250,6 +252,54 @@ test('every lease-route command skips sessionless provider-device resolution', ( } }); +test('takeover passes lease admission and uses the normal execution lock', () => { + assert.equal(isLeaseAdmissionExempt(INTERNAL_COMMANDS.humanControl), false); + assert.equal(shouldLockSessionExecution(INTERNAL_COMMANDS.humanControl), true); +}); + +test('human-control admission derives existing semantics and treats unclassified requests as mutations', () => { + for (const command of [ + 'snapshot', + 'screenshot', + 'get', + 'is', + 'logs', + 'network', + 'events', + 'audio', + 'trace', + 'devices', + 'apps', + 'appstate', + 'doctor', + 'human_control', + 'lease_heartbeat', + ]) { + assert.equal(isHumanControlMutation(makeRequest(command)), false, command); + } + for (const [command, positionals] of [ + ['clipboard', ['read']], + ['keyboard', ['status']], + ['alert', ['get']], + ['find', ['text', 'Save', 'get', 'text']], + ] as const) { + assert.equal(isHumanControlMutation(makeRequest(command, [...positionals])), false, command); + } + for (const [command, positionals] of [ + ['clipboard', ['write', 'value']], + ['clipboard', []], + ['keyboard', ['dismiss']], + ['alert', ['accept']], + ['find', ['text', 'Save', 'click']], + ['click', []], + ['viewport', []], + ['lease_release', []], + ['future-command', []], + ] as const) { + assert.equal(isHumanControlMutation(makeRequest(command, [...positionals])), true, command); + } +}); + function makeRequest(command: string, positionals: string[] = []): DaemonRequest { return { command, diff --git a/src/daemon/__tests__/device-mutation-drain.test.ts b/src/daemon/__tests__/device-mutation-drain.test.ts new file mode 100644 index 000000000..6c5a76cde --- /dev/null +++ b/src/daemon/__tests__/device-mutation-drain.test.ts @@ -0,0 +1,54 @@ +import { createControlLatch } from './human-control-fixtures.ts'; +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { DeviceMutationDrain } from '../device-mutation-drain.ts'; +import { getEventListeners } from 'node:events'; + +test('drain counts concurrent operations, releases on failure, and isolates device keys', async () => { + const drain = new DeviceMutationDrain(); + const first = createControlLatch(); + const second = createControlLatch(); + const one = drain.run('device-a', () => first.promise); + const two = drain.run('device-a', () => second.promise); + let idle = false; + const waited = drain.wait('device-a').then(() => { + idle = true; + }); + await drain.wait('device-b'); + first.resolve(); + await one; + assert.equal(idle, false); + const rejected = assert.rejects(two, /failure/); + second.reject(new Error('failure')); + await rejected; + await waited; + assert.equal(idle, true); + await drain.wait('device-a'); +}); + +test('canceling a drain waiter detaches its signal without canceling mutations or other waiters', async () => { + const drain = new DeviceMutationDrain(); + const finish = createControlLatch(); + const mutation = drain.run('device-a', () => finish.promise); + const controller = new AbortController(); + const reason = new Error('canceled waiter'); + const rejected = assert.rejects( + drain.wait('device-a', controller.signal), + (error) => error === reason, + ); + let drained = false; + const survivor = drain.wait('device-a').then(() => { + drained = true; + }); + controller.abort(reason); + await rejected; + assert.equal(drained, false); + assert.equal(getEventListeners(controller.signal, 'abort').length, 0); + const successController = new AbortController(); + const successful = drain.wait('device-a', successController.signal); + finish.resolve(); + await Promise.all([mutation, survivor, successful]); + assert.equal(drained, true); + assert.equal(getEventListeners(successController.signal, 'abort').length, 0); + await assert.rejects(drain.wait('device-a', controller.signal), (error) => error === reason); +}); diff --git a/src/daemon/__tests__/human-control-fixtures.ts b/src/daemon/__tests__/human-control-fixtures.ts new file mode 100644 index 000000000..8430d2537 --- /dev/null +++ b/src/daemon/__tests__/human-control-fixtures.ts @@ -0,0 +1,72 @@ +import type { HumanControlHold, HumanControlHoldScope } from '@agent-device/contracts/client'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { AllocateLeaseRequest } from '../lease-registry-scope.ts'; +import type { DaemonRequest } from '../types.ts'; +import { AppError } from '@agent-device/kernel/errors'; + +export const HUMAN_CONTROL_SCOPE: HumanControlHoldScope = { + backend: 'ios-instance', + leaseProvider: 'proxy', + deviceKey: 'ios:mobile:sim-1', +}; + +export const HUMAN_CONTROL_LEASE_REQUEST: AllocateLeaseRequest = { + tenantId: 'tenant-a', + runId: 'run-a', + clientId: 'client-a', + leaseBackend: HUMAN_CONTROL_SCOPE.backend, + leaseProvider: HUMAN_CONTROL_SCOPE.leaseProvider, + deviceKey: HUMAN_CONTROL_SCOPE.deviceKey, +}; + +export const HUMAN_CONTROL_HOLD: HumanControlHold = { + id: 'operator-1', + scope: HUMAN_CONTROL_SCOPE, + state: 'active', + reason: 'Manual inspection', + createdAt: 1_000, + updatedAt: 1_000, + expiresAt: 16_000, +}; + +export function humanControlRequest( + lease: DeviceLease, + command = 'human_control', + positionals = ['list'], +): DaemonRequest { + return { + token: 'test-token', + session: 'takeover-test', + command, + positionals, + flags: {}, + meta: { + sessionIsolation: 'tenant', + tenantId: lease.tenantId, + runId: lease.runId, + leaseId: lease.leaseId, + leaseBackend: lease.backend, + leaseProvider: lease.leaseProvider, + deviceKey: lease.deviceKey, + clientId: lease.clientId, + }, + }; +} + +export function isHumanControlError(error: unknown): boolean { + return ( + error instanceof AppError && + error.code === 'DEVICE_IN_USE' && + error.details?.reason === 'human_control_active' + ); +} + +export function createControlLatch() { + let resolve: () => void = () => {}; + let reject: (error: Error) => void = () => {}; + const promise = new Promise((onResolve, onReject) => { + resolve = onResolve; + reject = onReject; + }); + return { promise, resolve, reject }; +} diff --git a/src/daemon/__tests__/human-control-http.test.ts b/src/daemon/__tests__/human-control-http.test.ts new file mode 100644 index 000000000..326f2061f --- /dev/null +++ b/src/daemon/__tests__/human-control-http.test.ts @@ -0,0 +1,186 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { test, vi } from 'vitest'; +import { + closeLoopbackServer, + listenOnLoopback, + skipWhenLoopbackUnavailable, +} from '../../__tests__/test-utils/loopback.ts'; +import { HUMAN_CONTROL_HTTP_PREFIX } from '../human-control-contract.ts'; +import { LeaseRegistry } from '../lease-registry.ts'; +import { + HUMAN_CONTROL_SCOPE, + humanControlRequest, + createControlLatch, +} from './human-control-fixtures.ts'; +import { createHumanControlHarness } from './human-control-router-fixture.ts'; +import { tryHandleHumanControlHttpRoute } from '../human-control-http.ts'; +import { createDaemonHttpServer } from '../server/http-server.ts'; + +test('malformed request URLs return a normalized error', async () => { + let responseBody = ''; + let finishResponse: (() => void) | undefined; + const responseFinished = new Promise((resolve) => { + finishResponse = resolve; + }); + const req = { + url: 'http://[', + method: 'PUT', + headers: { authorization: 'Bearer daemon-secret' }, + } as http.IncomingMessage; + const res = { + statusCode: 0, + setHeader: () => undefined, + end: (body: string) => { + responseBody = body; + finishResponse?.(); + }, + } as unknown as http.ServerResponse; + + assert.equal( + tryHandleHumanControlHttpRoute({ + req, + res, + expectedToken: 'daemon-secret', + registry: new LeaseRegistry(), + }), + true, + ); + await responseFinished; + assert.equal(res.statusCode, 400); + assert.equal((JSON.parse(responseBody) as { code?: string }).code, 'INVALID_ARGS'); +}); + +for (const transport of ['tenant RPC', 'host PUT'] as const) { + test(`${transport} disconnect during drain removes its pending hold before the mutation ends`, async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const { registry, lease, handleRequest } = createHumanControlHarness(); + const finish = createControlLatch(); + const disconnected = createControlLatch(); + let mutationFinished = false; + const mutation = registry.runDeviceMutation(lease, async () => { + await finish.promise; + mutationFinished = true; + }); + const server = await createDaemonHttpServer({ + token: 'test-token', + leaseRegistry: registry, + handleRequest, + }); + server.on('request', (_req, res) => { + res.once('close', () => { + if (!res.writableFinished) disconnected.resolve(); + }); + }); + let request: http.ClientRequest | undefined; + try { + const port = await listenOnLoopback(server); + const isRpc = transport === 'tenant RPC'; + const body = JSON.stringify( + isRpc + ? { + jsonrpc: '2.0', + id: 'disconnected-takeover', + method: 'agent_device.command', + params: humanControlRequest(lease, 'human_control', ['put', 'disconnected', '{}']), + } + : { scope: HUMAN_CONTROL_SCOPE }, + ); + request = http.request({ + host: '127.0.0.1', + port, + path: isRpc ? '/rpc' : `${HUMAN_CONTROL_HTTP_PREFIX}/disconnected`, + method: isRpc ? 'POST' : 'PUT', + headers: { + authorization: 'Bearer test-token', + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + }, + }); + request.on('error', () => undefined); + request.end(body); + await vi.waitFor(() => { + assert.equal(registry.listHumanControlHolds({ kind: 'host' })[0]?.state, 'activating'); + }); + request.destroy(); + await disconnected.promise; + await vi.waitFor(() => { + assert.deepEqual(registry.listHumanControlHolds({ kind: 'host' }), []); + }); + assert.equal(mutationFinished, false); + finish.resolve(); + await mutation; + assert.deepEqual(registry.listHumanControlHolds({ kind: 'host' }), []); + assert.equal(await registry.runDeviceMutation(lease, async () => 'resumed'), 'resumed'); + } finally { + request?.destroy(); + finish.resolve(); + await mutation; + await closeLoopbackServer(server); + } + }); +} + +test('host administration and tenant RPC use the same lease registry with distinct authority', async (t) => { + if (await skipWhenLoopbackUnavailable(t)) return; + const { registry, lease, handleRequest } = createHumanControlHarness(); + const server = await createDaemonHttpServer({ + token: 'test-token', + leaseRegistry: registry, + handleRequest, + }); + try { + const port = await listenOnLoopback(server); + const origin = `http://127.0.0.1:${String(port)}`; + const baseUrl = origin + HUMAN_CONTROL_HTTP_PREFIX; + const headers = { authorization: 'Bearer test-token', 'content-type': 'application/json' }; + const denied = await fetch(baseUrl, { headers: { authorization: 'Bearer tenant-credential' } }); + assert.equal(denied.status, 401); + const invalid = await fetch(baseUrl + '/console', { + method: 'PUT', + headers, + body: JSON.stringify({ scope: { deviceKey: 'sim-1' } }), + }); + assert.equal(invalid.status, 400); + const created = await fetch(baseUrl + '/host', { + method: 'PUT', + headers, + body: JSON.stringify({ scope: HUMAN_CONTROL_SCOPE }), + }); + assert.equal(created.status, 200); + const body = (await created.json()) as { state: string; hold: { scope: unknown } }; + assert.equal(body.state, 'active'); + assert.deepEqual(body.hold.scope, HUMAN_CONTROL_SCOPE); + + const rpc = async (command: string, positionals: string[]) => { + const request = humanControlRequest(lease, command, positionals); + return await fetch(origin + '/rpc', { + method: 'POST', + headers, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'takeover-rpc', + method: 'agent_device.command', + params: request, + }), + }); + }; + assert.equal((await rpc('human_control', ['put', 'tenant', '{}'])).status, 200); + const blocked = await rpc('click', ['10', '10']); + assert.equal(blocked.status, 423); + assert.match(JSON.stringify(await blocked.json()), /human_control_active/); + assert.equal((await rpc('snapshot', [])).status, 200); + assert.equal((await rpc('human_control', ['remove', 'host'])).status, 401); + assert.equal((await rpc('human_control', ['remove', 'tenant'])).status, 200); + assert.equal((await rpc('click', ['10', '10'])).status, 423); + + const listed = await fetch(baseUrl, { headers }); + assert.equal(listed.status, 200); + assert.equal(((await listed.json()) as { holds: unknown[] }).holds.length, 1); + const released = await fetch(baseUrl + '/host', { method: 'DELETE', headers }); + assert.equal(released.status, 200); + assert.deepEqual(registry.listHumanControlHolds({ kind: 'host' }), []); + } finally { + await closeLoopbackServer(server); + } +}); diff --git a/src/daemon/__tests__/human-control-router-fixture.ts b/src/daemon/__tests__/human-control-router-fixture.ts new file mode 100644 index 000000000..8cbc7ee6d --- /dev/null +++ b/src/daemon/__tests__/human-control-router-fixture.ts @@ -0,0 +1,32 @@ +import { createTestDeviceInventoryGateways } from '../../__tests__/test-utils/device-inventory-gateways.ts'; +import { makeIosAppSession } from '../../__tests__/test-utils/session-factories.ts'; +import { makeSessionStore } from '../../__tests__/test-utils/store-factory.ts'; +import { buildSessionLeaseFromRequest } from '../lease-context.ts'; +import { LeaseRegistry } from '../lease-registry.ts'; +import { createRequestHandler } from '../request-router.ts'; +import { tenantScopedSessionName } from '../session-tenant-scope.ts'; +import { lifecycleDeviceRuntimeGateway } from './test-device-runtime-gateway.ts'; +import { HUMAN_CONTROL_LEASE_REQUEST, humanControlRequest } from './human-control-fixtures.ts'; + +export function createHumanControlHarness() { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const sessionStore = makeSessionStore('agent-device-human-control-'); + const sessionName = tenantScopedSessionName(lease.tenantId, 'takeover-test'); + sessionStore.set( + sessionName, + makeIosAppSession(sessionName, { + lease: buildSessionLeaseFromRequest(humanControlRequest(lease), lease), + }), + ); + const handleRequest = createRequestHandler({ + logPath: '/tmp/agent-device-human-control.log', + token: 'test-token', + sessionStore, + leaseRegistry: registry, + deviceInventoryGateways: createTestDeviceInventoryGateways(), + deviceRuntimeGateway: lifecycleDeviceRuntimeGateway, + trackDownloadableArtifact: () => 'artifact-1', + }); + return { registry, lease, sessionStore, sessionName, handleRequest }; +} diff --git a/src/daemon/__tests__/lease-registry-scope.test.ts b/src/daemon/__tests__/lease-registry-scope.test.ts new file mode 100644 index 000000000..1bb76af55 --- /dev/null +++ b/src/daemon/__tests__/lease-registry-scope.test.ts @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + createLeaseTtlResolver, + leaseDeviceBindingKey, + normalizeAllocateLeaseRequest, +} from '../lease-registry-scope.ts'; +import { HUMAN_CONTROL_LEASE_REQUEST, HUMAN_CONTROL_SCOPE } from './human-control-fixtures.ts'; + +test('allocation and human control share the exact contention identity', () => { + const scope = normalizeAllocateLeaseRequest(HUMAN_CONTROL_LEASE_REQUEST); + assert.equal(leaseDeviceBindingKey(scope), leaseDeviceBindingKey(HUMAN_CONTROL_SCOPE)); + assert.notEqual( + leaseDeviceBindingKey(scope), + leaseDeviceBindingKey({ ...scope, leaseProvider: 'other' }), + ); + assert.notEqual( + leaseDeviceBindingKey(scope), + leaseDeviceBindingKey({ ...scope, backend: 'ios-simulator' }), + ); + assert.notEqual( + leaseDeviceBindingKey(scope), + leaseDeviceBindingKey({ ...scope, deviceKey: 'sim-1' }), + ); + assert.equal(leaseDeviceBindingKey({ backend: 'ios-simulator' }), undefined); +}); + +test('lease TTL normalization retains defaults, limits, and invalid configuration handling', () => { + const defaults = createLeaseTtlResolver({}); + assert.equal(defaults(undefined), 60_000); + assert.equal(defaults(1.5), 60_000); + assert.equal(defaults(5_000), 5_000); + assert.equal(defaults(600_000), 600_000); + for (const ttl of [4_999, 600_001]) assert.throws(() => defaults(ttl), { code: 'INVALID_ARGS' }); + const configured = createLeaseTtlResolver({ + defaultLeaseTtlMs: 0, + minLeaseTtlMs: 0, + maxLeaseTtlMs: -1, + }); + assert.equal(configured(undefined), 1); + assert.equal(configured(1), 1); + assert.throws(() => configured(2), { code: 'INVALID_ARGS' }); +}); diff --git a/src/daemon/__tests__/lease-registry.test.ts b/src/daemon/__tests__/lease-registry.test.ts index 1fc4c0820..61dbffbe6 100644 --- a/src/daemon/__tests__/lease-registry.test.ts +++ b/src/daemon/__tests__/lease-registry.test.ts @@ -1,6 +1,13 @@ import { test } from 'vitest'; import assert from 'node:assert/strict'; +import { createRequestCanceledError } from '@agent-device/kernel/errors'; import { LeaseRegistry } from '../lease-registry.ts'; +import { + HUMAN_CONTROL_LEASE_REQUEST, + HUMAN_CONTROL_SCOPE, + isHumanControlError, + createControlLatch, +} from './human-control-fixtures.ts'; test('allocateLease creates lease and enforces tenant/run validation', () => { const registry = new LeaseRegistry(); @@ -355,3 +362,205 @@ function captureThrown(task: () => unknown): unknown { return error; } } + +test('human holds protect leases and release refreshes the original lease TTL atomically', async () => { + let now = 1_000; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 5_000 }); + const lease = registry.allocateLease({ ...HUMAN_CONTROL_LEASE_REQUEST, ttlMs: 10_000 }); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + await registry.putHumanControlHold(authority, 'console', {}); + now = 25_000; + assert.deepEqual(registry.consumeExpiredLeases(), []); + assert.equal(registry.consumeExpiredLease(lease.leaseId), undefined); + assert.equal(registry.listActiveLeases()[0]?.leaseId, lease.leaseId); + assert.throws( + () => registry.allocateLease({ ...HUMAN_CONTROL_LEASE_REQUEST, runId: 'run-b' }), + isHumanControlError, + ); + registry.removeHumanControlHold(authority, 'console'); + assert.equal(registry.listActiveLeases()[0]?.expiresAt, 35_000); + now = 35_000; + assert.equal(registry.consumeExpiredLeases()[0]?.leaseId, lease.leaseId); +}); + +test('hold expiry refreshes from the expiry instant, without reviving abandoned leases', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 5_000 }); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + await registry.putHumanControlHold(authority, 'console', { ttlMs: 10_000 }); + now = 9_000; + assert.equal(registry.listActiveLeases().length, 1); + now = 10_000; + assert.deepEqual(registry.listHumanControlHolds(authority), []); + assert.equal(registry.listActiveLeases()[0]?.expiresAt, 15_000); + now = 15_000; + assert.deepEqual(registry.listActiveLeases(), []); +}); + +test('heartbeat and overlapping holds preserve the lease until the final hold ends', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 5_000 }); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + await registry.putHumanControlHold(authority, 'first', { ttlMs: 10_000 }); + await registry.putHumanControlHold(authority, 'second', { ttlMs: 10_000 }); + now = 7_000; + const renewed = await registry.putHumanControlHold(authority, 'second', { ttlMs: 10_000 }); + assert.equal(renewed.createdAt, 0); + assert.equal(renewed.expiresAt, 17_000); + registry.removeHumanControlHold(authority, 'first'); + now = 12_000; + assert.throws(() => registry.assertHumanControlAdmission(lease), isHumanControlError); + assert.equal(registry.consumeExpiredLease(lease.leaseId), undefined); + now = 17_000; + assert.doesNotThrow(() => registry.assertHumanControlAdmission(lease)); + assert.equal(registry.listActiveLeases()[0]?.expiresAt, 22_000); +}); + +test('human control uses the lease backend/provider/device key, never aliases', async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + await registry.putHumanControlHold({ kind: 'host' }, 'console', { scope: HUMAN_CONTROL_SCOPE }); + assert.throws(() => registry.assertHumanControlAdmission(lease), isHumanControlError); + for (const scope of [ + { ...HUMAN_CONTROL_SCOPE, deviceKey: 'sim-1' }, + { ...HUMAN_CONTROL_SCOPE, deviceKey: 'ios:mobile:SIM-1' }, + { ...HUMAN_CONTROL_SCOPE, leaseProvider: 'another-provider' }, + { ...HUMAN_CONTROL_SCOPE, backend: 'ios-simulator' as const }, + ]) { + assert.doesNotThrow(() => registry.assertHumanControlAdmission(scope)); + } +}); + +test('lease owners cannot select another device or alter host/other-lease holds', async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const other = registry.allocateLease({ + ...HUMAN_CONTROL_LEASE_REQUEST, + deviceKey: 'other', + runId: 'run-b', + }); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + await registry.putHumanControlHold({ kind: 'host' }, 'host', { scope: HUMAN_CONTROL_SCOPE }); + await registry.putHumanControlHold({ kind: 'lease', leaseId: other.leaseId }, 'other', {}); + await assert.rejects( + registry.putHumanControlHold(authority, 'spoofed', { scope: HUMAN_CONTROL_SCOPE }), + { code: 'INVALID_ARGS' }, + ); + for (const id of ['host', 'other']) { + await assert.rejects(registry.putHumanControlHold(authority, id, {}), { code: 'UNAUTHORIZED' }); + assert.throws(() => registry.removeHumanControlHold(authority, id), { code: 'UNAUTHORIZED' }); + } + assert.deepEqual( + registry.listHumanControlHolds(authority).map((hold) => hold.id), + ['host'], + ); +}); + +test('activation drains admitted mutations, fences new ones, and starts TTL after draining', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now }); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + const finished = createControlLatch(); + const mutation = registry.runDeviceMutation(lease, () => finished.promise); + let active = false; + const activation = registry + .putHumanControlHold(authority, 'console', { ttlMs: 1_000 }) + .then((hold) => { + active = true; + return hold; + }); + await Promise.resolve(); + assert.equal(active, false); + assert.equal(registry.listHumanControlHolds(authority)[0]?.state, 'activating'); + await assert.rejects( + registry.runDeviceMutation(lease, async () => 'late mutation'), + isHumanControlError, + ); + now = 20_000; + finished.resolve(); + await mutation; + const hold = await activation; + assert.equal(active, true); + assert.equal(hold.expiresAt, 21_000); + registry.removeHumanControlHold(authority, 'console'); + assert.equal(await registry.runDeviceMutation(lease, async () => 'resumed'), 'resumed'); +}); + +test('release during activation cannot report a removed hold as active', async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + const finished = createControlLatch(); + const mutation = registry.runDeviceMutation(lease, () => finished.promise); + const activation = registry.putHumanControlHold(authority, 'console', {}); + registry.removeHumanControlHold(authority, 'console'); + const rejected = assert.rejects(activation, { code: 'COMMAND_FAILED' }); + finished.resolve(); + await mutation; + await rejected; + assert.deepEqual(registry.listHumanControlHolds(authority), []); +}); + +test('holds do not survive registry restart, including host holds created without a lease', async () => { + const registry = new LeaseRegistry(); + await registry.putHumanControlHold({ kind: 'host' }, 'console', { scope: HUMAN_CONTROL_SCOPE }); + assert.throws(() => registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST), isHumanControlError); + const restarted = new LeaseRegistry(); + assert.deepEqual(restarted.listHumanControlHolds({ kind: 'host' }), []); + assert.doesNotThrow(() => restarted.allocateLease(HUMAN_CONTROL_LEASE_REQUEST)); +}); + +test('canceled activation refreshes the protected lease without waiting for the mutation', async () => { + let now = 0; + const registry = new LeaseRegistry({ now: () => now, defaultLeaseTtlMs: 5_000 }); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + const finish = createControlLatch(); + const mutation = registry.runDeviceMutation(lease, () => finish.promise); + const controller = new AbortController(); + const activation = registry.putHumanControlHold(authority, 'console', {}, controller.signal); + const canceled = createRequestCanceledError(); + const rejected = assert.rejects(activation, (error) => error === canceled); + now = 20_000; + controller.abort(canceled); + await rejected; + assert.deepEqual(registry.listHumanControlHolds(authority), []); + assert.equal(registry.listActiveLeases()[0]?.expiresAt, 25_000); + finish.resolve(); + await mutation; + assert.deepEqual(registry.listHumanControlHolds(authority), []); +}); + +test('canceling a superseded activation cannot remove its successor or another hold', async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + const finish = createControlLatch(); + const mutation = registry.runDeviceMutation(lease, () => finish.promise); + const controller = new AbortController(); + const canceled = createRequestCanceledError(); + const rejected = assert.rejects( + registry.putHumanControlHold(authority, 'console', {}, controller.signal), + (error) => error === canceled, + ); + const successor = registry.putHumanControlHold(authority, 'console', { reason: 'successor' }); + const other = registry.putHumanControlHold(authority, 'other', {}); + controller.abort(canceled); + await rejected; + assert.deepEqual( + registry.listHumanControlHolds(authority).map((hold) => hold.id), + ['console', 'other'], + ); + finish.resolve(); + await mutation; + assert.equal((await successor).reason, 'successor'); + assert.equal((await other).state, 'active'); + await assert.rejects( + registry.putHumanControlHold(authority, 'console', {}, controller.signal), + (error) => error === canceled, + ); + assert.equal(registry.listHumanControlHolds(authority)[0]?.reason, 'successor'); +}); diff --git a/src/daemon/__tests__/request-handler-catalog.test.ts b/src/daemon/__tests__/request-handler-catalog.test.ts index e58c966fb..609f6aefb 100644 --- a/src/daemon/__tests__/request-handler-catalog.test.ts +++ b/src/daemon/__tests__/request-handler-catalog.test.ts @@ -29,6 +29,7 @@ import { createAudioProbeAdmissionLedger } from '../audio-probe-admission-ledger import { createPerfCaptureAdmissionLedger } from '../perf-capture-admission-ledger.ts'; const SPECIALIZED_ROUTES = [ + 'humanControl', 'lease', 'session', 'snapshot', diff --git a/src/daemon/daemon-command-registry.ts b/src/daemon/daemon-command-registry.ts index 7c97d2221..9c9c0f4b8 100644 --- a/src/daemon/daemon-command-registry.ts +++ b/src/daemon/daemon-command-registry.ts @@ -4,7 +4,11 @@ import { type SessionCommandKind, } from '../core/command-descriptor/daemon-command-descriptor.ts'; import { deriveDaemonCommandDescriptors } from '../core/command-descriptor/derive.ts'; -import { commandDescriptors } from '../core/command-descriptor/registry.ts'; +import { + commandDescriptors, + resolveCommandRecordingEffect, + resolveCommandDeviceClaimPolicy, +} from '../core/command-descriptor/registry.ts'; import type { RefFrameEffect } from '@agent-device/contracts/replay'; import type { DaemonRequest } from './types.ts'; @@ -74,6 +78,14 @@ export function shouldGuardAndroidBlockingDialog(command: string): boolean { return getDaemonCommandDescriptor(command)?.androidBlockingDialogGuard === true; } +export function isHumanControlMutation(req: DaemonRequest): boolean { + if (req.command === 'human_control' || req.command === 'lease_heartbeat') return false; + const recordingEffect = resolveCommandRecordingEffect(req); + if (recordingEffect !== undefined) return recordingEffect !== 'observes-app'; + if (getSessionCommandKind(req.command) === 'observability') return false; + return resolveCommandDeviceClaimPolicy(req.command) !== 'observe'; +} + export function shouldPreferExplicitDeviceOverExistingSession(req: DaemonRequest): boolean { return getDaemonCommandDescriptor(req.command)?.preferExplicitDeviceOverExistingSession === true; } diff --git a/src/daemon/device-mutation-drain.ts b/src/daemon/device-mutation-drain.ts new file mode 100644 index 000000000..a37befe5f --- /dev/null +++ b/src/daemon/device-mutation-drain.ts @@ -0,0 +1,42 @@ +export class DeviceMutationDrain { + private readonly active = new Map(); + private readonly waiters = new Map void>>(); + + async run(key: string, task: () => Promise): Promise { + this.active.set(key, (this.active.get(key) ?? 0) + 1); + try { + return await task(); + } finally { + const remaining = (this.active.get(key) ?? 1) - 1; + if (remaining > 0) { + this.active.set(key, remaining); + } else { + this.active.delete(key); + const waiters = this.waiters.get(key); + this.waiters.delete(key); + for (const resolve of waiters ?? []) resolve(); + } + } + } + + async wait(key: string, signal?: AbortSignal): Promise { + signal?.throwIfAborted(); + if (!this.active.has(key)) return; + const waiters = this.waiters.get(key) ?? new Set<() => void>(); + let drained: () => void = () => {}; + let aborted: () => void = () => {}; + try { + await new Promise((resolve, reject) => { + drained = resolve; + aborted = () => reject(signal?.reason); + waiters.add(drained); + this.waiters.set(key, waiters); + signal?.addEventListener('abort', aborted, { once: true }); + }); + } finally { + signal?.removeEventListener('abort', aborted); + waiters.delete(drained); + if (waiters.size === 0 && this.waiters.get(key) === waiters) this.waiters.delete(key); + } + } +} diff --git a/src/daemon/handlers/__tests__/human-control.test.ts b/src/daemon/handlers/__tests__/human-control.test.ts new file mode 100644 index 000000000..ea816cfd5 --- /dev/null +++ b/src/daemon/handlers/__tests__/human-control.test.ts @@ -0,0 +1,137 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { createRequestExecutionScope } from '../../request-execution-scope.ts'; +import { + HUMAN_CONTROL_LEASE_REQUEST, + HUMAN_CONTROL_SCOPE, + humanControlRequest, + isHumanControlError, + createControlLatch, +} from '../../__tests__/human-control-fixtures.ts'; +import { createHumanControlHarness } from '../../__tests__/human-control-router-fixture.ts'; + +test('lease-owner takeover uses the production admission gate, preserves the session, and resumes after release', async () => { + const { registry, lease, handleRequest, sessionStore, sessionName } = createHumanControlHarness(); + const activate = await handleRequest( + humanControlRequest(lease, 'human_control', ['put', 'console', '{}']), + ); + assert.equal(activate.ok, true); + const blocked = await handleRequest(humanControlRequest(lease, 'click', ['10', '10'])); + assert.equal(blocked.ok, false); + if (blocked.ok) throw new Error('Expected blocked mutation'); + assert.equal(blocked.error.code, 'DEVICE_IN_USE'); + assert.equal(blocked.error.details?.reason, 'human_control_active'); + assert.equal(blocked.error.retriable, true); + assert.equal((await handleRequest(humanControlRequest(lease, 'snapshot', []))).ok, true); + assert.equal((await handleRequest(humanControlRequest(lease, 'lease_heartbeat', []))).ok, true); + assert.ok(sessionStore.get(sessionName)); + const release = await handleRequest( + humanControlRequest(lease, 'human_control', ['remove', 'console']), + ); + assert.equal(release.ok, true); + assert.deepEqual(registry.listHumanControlHolds({ kind: 'host' }), []); + const scope = await createRequestExecutionScope({ + req: humanControlRequest(lease, 'click', []), + sessionStore, + leaseRegistry: registry, + }); + assert.equal(await scope.runLocked(async () => 'resumed'), 'resumed'); +}); + +test('takeover refuses missing, expired, and foreign lease ownership before changing state', async () => { + const { registry, lease, handleRequest } = createHumanControlHarness(); + const valid = humanControlRequest(lease, 'human_control', ['put', 'console', '{}']); + for (const patch of [ + { tenantId: 'tenant-b' }, + { runId: 'run-b' }, + { clientId: 'client-b' }, + { deviceKey: 'other-device' }, + { leaseId: 'aaaaaaaaaaaaaaaa' }, + { leaseProvider: 'other-provider' }, + ]) { + const response = await handleRequest({ ...valid, meta: { ...valid.meta, ...patch } }); + assert.equal(response.ok, false, JSON.stringify(patch)); + assert.deepEqual(registry.listHumanControlHolds({ kind: 'host' }), []); + } + const local = await handleRequest({ ...valid, meta: undefined, session: 'local-no-lease' }); + assert.equal(local.ok, false); + if (!local.ok) assert.equal(local.error.code, 'UNSUPPORTED_OPERATION'); + registry.releaseLease({ ...HUMAN_CONTROL_LEASE_REQUEST, leaseId: lease.leaseId }); + assert.equal((await handleRequest(valid)).ok, false); +}); + +test('admitted tenants cannot retarget takeover or release provider-host holds', async () => { + const { registry, lease, handleRequest } = createHumanControlHarness(); + await registry.putHumanControlHold({ kind: 'host' }, 'host', { scope: HUMAN_CONTROL_SCOPE }); + const injected = await handleRequest( + humanControlRequest(lease, 'human_control', [ + 'put', + 'tenant', + JSON.stringify({ scope: { ...HUMAN_CONTROL_SCOPE, deviceKey: 'other' } }), + ]), + ); + assert.equal(injected.ok, false); + const denied = await handleRequest( + humanControlRequest(lease, 'human_control', ['remove', 'host']), + ); + assert.equal(denied.ok, false); + if (!denied.ok) assert.equal(denied.error.code, 'UNAUTHORIZED'); + assert.equal(registry.listHumanControlHolds({ kind: 'host' }).length, 1); +}); + +test('a fresh-session mutation without an advisory device lock still drains before host activation', async () => { + const { registry, lease, sessionStore } = createHumanControlHarness(); + const req = { ...humanControlRequest(lease, 'click', []), session: 'fresh-session' }; + const scope = await createRequestExecutionScope({ req, sessionStore, leaseRegistry: registry }); + const started = createControlLatch(); + const finish = createControlLatch(); + const mutation = scope.runLocked(async () => { + started.resolve(); + await finish.promise; + }); + await started.promise; + let active = false; + const activation = registry + .putHumanControlHold({ kind: 'host' }, 'host', { scope: HUMAN_CONTROL_SCOPE }) + .then(() => { + active = true; + }); + await Promise.resolve(); + assert.equal(active, false); + const later = await createRequestExecutionScope({ + req: { ...req, session: 'another-fresh-session' }, + sessionStore, + leaseRegistry: registry, + }); + await assert.rejects( + later.runLocked(async () => 'must not run'), + isHumanControlError, + ); + finish.resolve(); + await mutation; + await activation; + assert.equal(active, true); +}); + +test('a nested mutation is stopped when takeover begins during its parent request', async () => { + const { registry, lease, sessionStore } = createHumanControlHarness(); + const req = humanControlRequest(lease, 'replay', []); + const scope = await createRequestExecutionScope({ req, sessionStore, leaseRegistry: registry }); + let activation: Promise | undefined; + await scope.runLocked(async () => { + activation = registry.putHumanControlHold({ kind: 'host' }, 'host', { + scope: HUMAN_CONTROL_SCOPE, + }); + const nested = await createRequestExecutionScope({ + req: humanControlRequest(lease, 'click', []), + sessionStore, + leaseRegistry: registry, + }); + await assert.rejects( + nested.runAdmitted(async () => 'must not run'), + isHumanControlError, + ); + }); + await activation; + assert.equal(registry.listHumanControlHolds({ kind: 'host' })[0]?.state, 'active'); +}); diff --git a/src/daemon/handlers/__tests__/lease-artifacts.test.ts b/src/daemon/handlers/__tests__/lease-artifacts.test.ts index abf0062ee..9306343ad 100644 --- a/src/daemon/handlers/__tests__/lease-artifacts.test.ts +++ b/src/daemon/handlers/__tests__/lease-artifacts.test.ts @@ -118,16 +118,18 @@ test('artifacts refuses an expired provider session after retention before lazy }); test('artifacts refuses a provider session returned after allocation expiry retention', async () => { - const clockValues = [1_000, 1_000, 1_099, 1_151] as const; - let clockIndex = 0; + let now = 1_000; const world = createWorld({ - now: () => clockValues[Math.min(clockIndex++, clockValues.length - 1)] ?? 1_151, + now: () => now, defaultLeaseTtlMs: 100, minLeaseTtlMs: 1, maxLeaseTtlMs: 100, providerSessionRetentionMs: 50, }); - world.lifecycle.allocate = async () => ({ providerSessionId: 'late-allocation-session' }); + world.lifecycle.allocate = async (lease) => { + now = lease.expiresAt + 51; + return { providerSessionId: 'late-allocation-session' }; + }; await allocateLease(world, 'tenant-a', 'run-a'); await assertProviderSessionNotOwned(world, { diff --git a/src/daemon/handlers/__tests__/lease.test.ts b/src/daemon/handlers/__tests__/lease.test.ts new file mode 100644 index 000000000..08fdb22c7 --- /dev/null +++ b/src/daemon/handlers/__tests__/lease.test.ts @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { handleLeaseCommands } from '../lease.ts'; +import { LeaseRegistry } from '../../lease-registry.ts'; +import { makeSessionStore } from '../../../__tests__/test-utils/store-factory.ts'; +import { AppError } from '@agent-device/kernel/errors'; +import { clearRequestCanceled, markRequestCanceled } from '@agent-device/host-kit/request'; +import { + HUMAN_CONTROL_LEASE_REQUEST, + HUMAN_CONTROL_SCOPE, + createControlLatch, + humanControlRequest, +} from '../../__tests__/human-control-fixtures.ts'; + +for (const operation of ['allocate', 'release'] as const) { + test(`host activation drains provider lease ${operation} before reporting active`, async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const started = createControlLatch(); + const finish = createControlLatch(); + const request = humanControlRequest(lease, `lease_${operation}`, []); + const mutation = handleLeaseCommands({ + req: request, + sessionName: request.session, + sessionStore: makeSessionStore('agent-device-held-provider-'), + leaseRegistry: registry, + leaseLifecycleProvider: { + [operation]: async () => { + started.resolve(); + await finish.promise; + return {}; + }, + }, + }); + await started.promise; + let active = false; + const activation = registry + .putHumanControlHold({ kind: 'host' }, 'host', { scope: HUMAN_CONTROL_SCOPE }) + .then(() => { + active = true; + }); + await Promise.resolve(); + assert.equal(active, false); + finish.resolve(); + assert.equal((await mutation)?.ok, true); + await activation; + assert.equal(active, true); + }); +} + +test('activation drains canceled provider allocation and its release cleanup', async () => { + const registry = new LeaseRegistry(); + const lease = registry.allocateLease(HUMAN_CONTROL_LEASE_REQUEST); + const started = createControlLatch(); + const finish = createControlLatch(); + const request = humanControlRequest(lease, 'lease_allocate', []); + const requestId = 'held-canceled-allocation'; + request.meta = { ...request.meta, requestId }; + let providerReleased = false; + const mutation = handleLeaseCommands({ + req: request, + sessionName: request.session, + sessionStore: makeSessionStore('agent-device-held-canceled-provider-'), + leaseRegistry: registry, + leaseLifecycleProvider: { + allocate: async () => { + started.resolve(); + await finish.promise; + markRequestCanceled(requestId); + return {}; + }, + release: async () => { + assert.equal(registry.listHumanControlHolds({ kind: 'host' })[0]?.state, 'activating'); + providerReleased = true; + return {}; + }, + }, + }); + await started.promise; + const activation = registry.putHumanControlHold({ kind: 'host' }, 'host', { + scope: HUMAN_CONTROL_SCOPE, + }); + finish.resolve(); + try { + await assert.rejects( + mutation, + (error: unknown) => error instanceof AppError && error.details?.released === true, + ); + await activation; + assert.equal(providerReleased, true); + assert.equal(registry.listActiveLeases().length, 0); + } finally { + clearRequestCanceled(requestId); + } +}); diff --git a/src/daemon/handlers/human-control.ts b/src/daemon/handlers/human-control.ts new file mode 100644 index 000000000..6ab31ba5f --- /dev/null +++ b/src/daemon/handlers/human-control.ts @@ -0,0 +1,67 @@ +import { AppError } from '@agent-device/kernel/errors'; +import { getRequestSignal } from '@agent-device/host-kit/request'; +import { parseHumanControlHoldInput } from '../human-control-contract.ts'; +import type { LeaseRegistry } from '../lease-registry.ts'; +import type { DaemonRequest, DaemonResponse } from '../types.ts'; + +export async function handleHumanControlCommand(params: { + req: DaemonRequest; + registry: LeaseRegistry; +}): Promise { + const { req, registry } = params; + const lease = req.internal?.admittedLease; + if (!lease) { + throw new AppError('UNAUTHORIZED', 'Human control requires an admitted remote lease.'); + } + const authority = { kind: 'lease', leaseId: lease.leaseId } as const; + const positionals = req.positionals ?? []; + const [action, holdId = '', rawInput = ''] = positionals; + switch (action) { + case 'list': + assertArgumentCount(positionals, 1); + return { ok: true, data: { holds: registry.listHumanControlHolds(authority) } }; + case 'put': { + assertArgumentCount(positionals, 3); + const hold = await registry.putHumanControlHold( + authority, + holdId, + readHoldInput(rawInput), + getRequestSignal(req.meta?.requestId), + ); + return { ok: true, data: { hold, state: 'active' } }; + } + case 'remove': { + assertArgumentCount(positionals, 2); + const hold = registry.removeHumanControlHold(authority, holdId); + return { ok: true, data: { released: Boolean(hold), ...(hold ? { hold } : {}) } }; + } + default: + throw invalidArguments(); + } +} + +function readHoldInput(raw: string) { + let input: unknown; + try { + input = JSON.parse(raw); + } catch (error) { + throw new AppError( + 'INVALID_ARGS', + 'Human-control payload must be valid JSON.', + undefined, + error, + ); + } + return parseHumanControlHoldInput(input); +} + +function assertArgumentCount(positionals: string[], expected: number): void { + if (positionals.length !== expected) throw invalidArguments(); +} + +function invalidArguments(): AppError { + return new AppError( + 'INVALID_ARGS', + 'human_control requires list, put , or remove .', + ); +} diff --git a/src/daemon/handlers/lease.ts b/src/daemon/handlers/lease.ts index 36117e310..6c14c8b11 100644 --- a/src/daemon/handlers/lease.ts +++ b/src/daemon/handlers/lease.ts @@ -9,7 +9,8 @@ import type { CloudArtifactProvider, } from '@agent-device/contracts/observability'; import type { DaemonRequest, DaemonResponse } from '../types.ts'; -import type { LeaseRegistry, ReleaseLeaseRequest } from '../lease-registry.ts'; +import type { LeaseRegistry } from '../lease-registry.ts'; +import type { ReleaseLeaseRequest } from '../lease-registry-scope.ts'; import type { SessionStore } from '../session-store.ts'; import { isProxyLeaseScope, @@ -70,27 +71,33 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise | undefined; - try { - providerData = await leaseLifecycleProvider?.allocate?.(lease, { - ...leaseLifecycleContext(req), - signal: getRequestSignal(req.meta?.requestId), - deadline: Date.now() + LEASE_ALLOCATION_BUDGET_MS, - }); - recordProviderSession(leaseRegistry, lease, providerData); - } catch (error) { - leaseRegistry.releaseLease(leaseReleaseRequestFor(lease)); - throw error; - } - if (isRequestCanceled(req.meta?.requestId)) { - // The requester left while the provider was allocating; the lease it - // produced is real (and billed) and nobody will ever release it. - throw await releaseAllocationForGoneRequester(lease, leaseLifecycleProvider, leaseRegistry); - } - return { - ok: true, - data: { lease, ...(providerData ? { provider: providerData } : {}) }, - }; + return await leaseRegistry.runDeviceMutation(lease, async () => { + let providerData: Record | undefined; + try { + providerData = await leaseLifecycleProvider?.allocate?.(lease, { + ...leaseLifecycleContext(req), + signal: getRequestSignal(req.meta?.requestId), + deadline: Date.now() + LEASE_ALLOCATION_BUDGET_MS, + }); + recordProviderSession(leaseRegistry, lease, providerData); + } catch (error) { + leaseRegistry.releaseLease(leaseReleaseRequestFor(lease)); + throw error; + } + if (isRequestCanceled(req.meta?.requestId)) { + // The requester left while the provider was allocating; the lease it + // produced is real (and billed) and nobody will ever release it. + throw await releaseAllocationForGoneRequester( + lease, + leaseLifecycleProvider, + leaseRegistry, + ); + } + return { + ok: true, + data: { lease, ...(providerData ? { provider: providerData } : {}) }, + }; + }); } case 'lease_heartbeat': { const lease = leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(leaseScope)); @@ -105,12 +112,17 @@ export async function handleLeaseCommands(args: LeaseHandlerArgs): Promise + await releaseLease( + leaseRegistry, + leaseLifecycleProvider, + lease, + releaseRequest, + leaseLifecycleContext(req), + ), ); return { ok: true, diff --git a/src/daemon/human-control-contract.ts b/src/daemon/human-control-contract.ts new file mode 100644 index 000000000..266260e32 --- /dev/null +++ b/src/daemon/human-control-contract.ts @@ -0,0 +1,103 @@ +import type { + HumanControlHold, + HumanControlHoldOptions, + HumanControlHoldScope, +} from '@agent-device/contracts/client'; +import { AppError } from '@agent-device/kernel/errors'; +import { + normalizeDeviceKey, + normalizeLeaseBackend, + normalizeLeaseProvider, +} from './lease-registry-scope.ts'; + +export const HUMAN_CONTROL_HTTP_PREFIX = '/admin/human-control/holds'; + +export type HumanControlHoldInput = HumanControlHoldOptions & { + scope?: HumanControlHoldScope; +}; + +export type HumanControlAuthority = { kind: 'host' } | { kind: 'lease'; leaseId: string }; + +export function parseHumanControlHoldInput(value: unknown): HumanControlHoldInput { + const record = readHumanControlRecord(value, 'Human-control request body'); + const reason = record.reason; + if (reason !== undefined && (typeof reason !== 'string' || reason.trim().length > 512)) { + throw new AppError('INVALID_ARGS', 'Human-control reason must be at most 512 characters.'); + } + const ttlMs = record.ttlMs; + if ( + ttlMs !== undefined && + (typeof ttlMs !== 'number' || !Number.isInteger(ttlMs) || ttlMs < 1_000 || ttlMs > 86_400_000) + ) { + throw new AppError('INVALID_ARGS', 'Human-control ttlMs must be between 1000 and 86400000.'); + } + return { + ...(record.scope === undefined ? {} : { scope: parseHumanControlScope(record.scope) }), + ...(typeof reason === 'string' && reason.trim() ? { reason: reason.trim() } : {}), + ...(typeof ttlMs === 'number' ? { ttlMs } : {}), + }; +} + +function parseHumanControlScope(value: unknown): HumanControlHoldScope { + const scope = readHumanControlRecord(value, 'Host human-control scope'); + if ( + typeof scope.backend !== 'string' || + !scope.backend.trim() || + typeof scope.deviceKey !== 'string' + ) { + throw new AppError( + 'INVALID_ARGS', + 'Host human control requires scope.backend and scope.deviceKey.', + ); + } + if (scope.leaseProvider !== undefined && typeof scope.leaseProvider !== 'string') { + throw new AppError('INVALID_ARGS', 'scope.leaseProvider must be a string.'); + } + const leaseProvider = normalizeLeaseProvider(scope.leaseProvider); + return { + backend: normalizeLeaseBackend(scope.backend), + deviceKey: normalizeDeviceKey(scope.deviceKey)!, + ...(leaseProvider ? { leaseProvider } : {}), + }; +} + +function readHumanControlRecord(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new AppError('INVALID_ARGS', `${label} must be an object.`); + } + return value as Record; +} + +export function normalizeHumanControlHoldId(id: string): string { + const value = typeof id === 'string' ? id.trim() : ''; + if (!/^[a-zA-Z0-9._-]{1,128}$/.test(value)) { + throw new AppError( + 'INVALID_ARGS', + 'Invalid human-control hold id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', + ); + } + return value; +} + +export function cloneHumanControlHold(hold: HumanControlHold): HumanControlHold { + return { ...hold, scope: { ...hold.scope } }; +} + +export function humanControlActiveError(hold: HumanControlHold): AppError { + return new AppError( + 'DEVICE_IN_USE', + 'A human is interacting with this simulator or device; agent interactions are temporarily disabled.', + { + reason: 'human_control_active', + blockedBy: 'human_control', + holdId: hold.id, + deviceKey: hold.scope.deviceKey, + pausedAt: new Date(hold.createdAt).toISOString(), + ...(hold.expiresAt === undefined + ? {} + : { expiresAt: new Date(hold.expiresAt).toISOString() }), + ...(hold.reason ? { humanReason: hold.reason } : {}), + hint: 'Wait until the human-control hold is released before retrying.', + }, + ); +} diff --git a/src/daemon/human-control-http.ts b/src/daemon/human-control-http.ts new file mode 100644 index 000000000..0f88d43ba --- /dev/null +++ b/src/daemon/human-control-http.ts @@ -0,0 +1,181 @@ +import type http from 'node:http'; +import { AppError, createRequestCanceledError, normalizeError } from '@agent-device/kernel/errors'; +import { readNodeHttpRequestBody } from '../utils/node-http.ts'; +import { timingSafeStringEqual } from '../utils/timing-safe-equal.ts'; +import { sendRestJsonError } from './http-errors.ts'; +import { + HUMAN_CONTROL_HTTP_PREFIX, + parseHumanControlHoldInput, + type HumanControlHoldInput, +} from './human-control-contract.ts'; +import type { LeaseRegistry } from './lease-registry.ts'; + +const MAX_HUMAN_CONTROL_BODY_BYTES = 16 * 1024; + +type HumanControlHttpRoute = + | { kind: 'list' } + | { kind: 'upsert'; holdId: string } + | { kind: 'remove'; holdId: string } + | { kind: 'invalid' } + | { kind: 'unsupported' }; + +type HumanControlHttpParams = { + req: http.IncomingMessage; + res: http.ServerResponse; + expectedToken: string; + registry: LeaseRegistry; +}; + +export function tryHandleHumanControlHttpRoute(params: HumanControlHttpParams): boolean { + const route = resolveHumanControlRoute(params.req); + if (!route) return false; + void handleHumanControlRoute(route, params); + return true; +} + +async function handleHumanControlRoute( + route: HumanControlHttpRoute, + params: HumanControlHttpParams, +): Promise { + const { req, res, expectedToken } = params; + try { + assertAuthorized(req, expectedToken); + await executeHumanControlRoute(route, params); + } catch (error) { + sendRestJsonError(res, normalizeError(error)); + } +} + +async function executeHumanControlRoute( + route: HumanControlHttpRoute, + params: HumanControlHttpParams, +): Promise { + switch (route.kind) { + case 'list': + sendJson(params.res, { + ok: true, + holds: params.registry.listHumanControlHolds({ kind: 'host' }), + }); + return; + case 'upsert': + await upsertHumanControlHold(route.holdId, params); + return; + case 'remove': + removeHumanControlHold(route.holdId, params); + return; + case 'unsupported': + sendMethodNotAllowed(params.res); + return; + case 'invalid': + throw new AppError('INVALID_ARGS', 'Invalid request URL.'); + } +} + +async function upsertHumanControlHold( + holdId: string, + params: HumanControlHttpParams, +): Promise { + const { req, res } = params; + const controller = new AbortController(); + const cancelIfDisconnected = () => { + if (!res.writableFinished) controller.abort(createRequestCanceledError()); + }; + req.once('aborted', cancelIfDisconnected); + res.once('close', cancelIfDisconnected); + if (req.aborted || res.destroyed) cancelIfDisconnected(); + try { + const input = await readHoldInput(req); + const hold = await params.registry.putHumanControlHold( + { kind: 'host' }, + holdId, + input, + controller.signal, + ); + sendJson(res, { ok: true, hold, state: 'active' }); + } finally { + req.off('aborted', cancelIfDisconnected); + res.off('close', cancelIfDisconnected); + } +} + +function removeHumanControlHold(holdId: string, params: HumanControlHttpParams): void { + const hold = params.registry.removeHumanControlHold({ kind: 'host' }, holdId); + sendJson(params.res, { ok: true, released: Boolean(hold), ...(hold ? { hold } : {}) }); +} + +function sendMethodNotAllowed(res: http.ServerResponse): void { + res.statusCode = 405; + res.setHeader('allow', 'GET, PUT, DELETE'); + sendJson(res, { ok: false, error: 'Method not allowed', code: 'INVALID_ARGS' }); +} + +function resolveHumanControlRoute(req: http.IncomingMessage): HumanControlHttpRoute | null { + const pathname = parseRequestPathname(req.url); + if (pathname === null) return { kind: 'invalid' }; + return resolveHumanControlPathRoute(pathname, req.method); +} + +function resolveHumanControlPathRoute( + pathname: string, + method: string | undefined, +): HumanControlHttpRoute | null { + if (pathname === HUMAN_CONTROL_HTTP_PREFIX) { + return method === 'GET' ? { kind: 'list' } : { kind: 'unsupported' }; + } + if (!pathname.startsWith(`${HUMAN_CONTROL_HTTP_PREFIX}/`)) return null; + const holdId = pathname.slice(HUMAN_CONTROL_HTTP_PREFIX.length + 1); + if (!holdId || holdId.includes('/')) return { kind: 'unsupported' }; + if (method === 'PUT') return { kind: 'upsert', holdId }; + if (method === 'DELETE') return { kind: 'remove', holdId }; + return { kind: 'unsupported' }; +} + +function parseRequestPathname(url: string | undefined): string | null { + try { + return new URL(url ?? '/', 'http://127.0.0.1').pathname; + } catch { + return null; + } +} + +async function readHoldInput(req: http.IncomingMessage): Promise { + const raw = await readNodeHttpRequestBody( + req, + MAX_HUMAN_CONTROL_BODY_BYTES, + 'Human-control request body is too large.', + ); + let parsed: unknown; + try { + parsed = JSON.parse(raw.toString('utf8')); + } catch (error) { + throw new AppError( + 'INVALID_ARGS', + 'Human-control request body must be valid JSON.', + undefined, + error, + ); + } + return parseHumanControlHoldInput(parsed); +} + +function assertAuthorized(req: http.IncomingMessage, expectedToken: string): void { + const authorization = + typeof req.headers.authorization === 'string' ? req.headers.authorization : ''; + const bearer = authorization.toLowerCase().startsWith('bearer ') + ? authorization.slice('bearer '.length) + : ''; + const headerToken = + typeof req.headers['x-agent-device-token'] === 'string' + ? req.headers['x-agent-device-token'] + : ''; + const token = headerToken || bearer; + if (!token || !timingSafeStringEqual(token, expectedToken)) { + throw new AppError('UNAUTHORIZED', 'Invalid token'); + } +} + +function sendJson(res: http.ServerResponse, body: Record): void { + res.statusCode ||= 200; + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(body)); +} diff --git a/src/daemon/lease-registry-scope.ts b/src/daemon/lease-registry-scope.ts new file mode 100644 index 000000000..614afd244 --- /dev/null +++ b/src/daemon/lease-registry-scope.ts @@ -0,0 +1,339 @@ +import crypto from 'node:crypto'; +import type { DeviceLease } from '@agent-device/contracts/device'; +import type { LeaseBackend } from '@agent-device/kernel/contracts'; +import { AppError } from '@agent-device/kernel/errors'; +import { normalizeTenantId } from './config.ts'; + +export type LeaseRegistryOptions = { + maxActiveSimulatorLeases?: number; + defaultLeaseTtlMs?: number; + minLeaseTtlMs?: number; + maxLeaseTtlMs?: number; + providerSessionRetentionMs?: number; + now?: () => number; + onLeaseExpired?: (lease: DeviceLease) => void; +}; + +export type AllocateLeaseRequest = { + tenantId: string; + runId: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; + ttlMs?: number; +}; + +export type HeartbeatLeaseRequest = { + leaseId: string; + tenantId?: string; + runId?: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; + ttlMs?: number; +}; + +export type ReleaseLeaseRequest = { + leaseId: string; + tenantId?: string; + runId?: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; +}; + +export type AdmissionRequest = { + tenantId?: string; + runId?: string; + leaseId?: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; +}; + +type LeaseScopeMatchRequest = { + tenantId?: string; + runId?: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; +}; + +type NormalizedLeaseScopeMatchRequest = { + tenantId?: string; + runId?: string; + leaseBackend?: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; +}; + +export type NormalizedAllocateLeaseRequest = { + tenantId: string; + runId: string; + backend: LeaseBackend; + leaseProvider?: string; + deviceKey?: string; + clientId?: string; + ttlMs?: number; +}; + +const DEFAULT_LEASE_TTL_MS = 60_000; +const MIN_LEASE_TTL_MS = 5_000; +const MAX_LEASE_TTL_MS = 10 * 60_000; +const DEFAULT_LEASE_PROVIDER = 'default'; + +export function createLeaseTtlResolver(options: LeaseRegistryOptions) { + const defaultTtl = Number.isInteger(options.defaultLeaseTtlMs) + ? Math.max(1, Number(options.defaultLeaseTtlMs)) + : DEFAULT_LEASE_TTL_MS; + const minTtl = Number.isInteger(options.minLeaseTtlMs) + ? Math.max(1, Number(options.minLeaseTtlMs)) + : MIN_LEASE_TTL_MS; + const maxTtl = Number.isInteger(options.maxLeaseTtlMs) + ? Math.max(minTtl, Number(options.maxLeaseTtlMs)) + : MAX_LEASE_TTL_MS; + return (raw: number | undefined): number => { + if (!Number.isInteger(raw)) return defaultTtl; + const value = Number(raw); + if (value < minTtl || value > maxTtl) { + throw new AppError('INVALID_ARGS', `Lease ttlMs must be between ${minTtl} and ${maxTtl}.`); + } + return value; + }; +} + +function normalizeRunId(raw: string | undefined): string | undefined { + if (!raw) return undefined; + const value = raw.trim(); + if (!value) return undefined; + if (!/^[a-zA-Z0-9._-]{1,128}$/.test(value)) return undefined; + return value; +} + +export function normalizeLeaseId(raw: string | undefined): string | undefined { + if (!raw) return undefined; + const value = raw.trim(); + if (!value) return undefined; + if (!/^[a-f0-9]{16,128}$/i.test(value)) return undefined; + return value.toLowerCase(); +} + +export function normalizeRequiredLeaseId(raw: string | undefined): string { + const leaseId = normalizeLeaseId(raw); + if (!leaseId) throw new AppError('INVALID_ARGS', 'Invalid lease id.'); + return leaseId; +} + +export function normalizeLeaseBackend(raw: string | undefined): LeaseBackend { + const value = (raw ?? '').trim().toLowerCase(); + if (!value || value === 'ios-simulator') return 'ios-simulator'; + if (value === 'ios-instance' || value === 'android-instance') return value; + throw new AppError('INVALID_ARGS', `Unsupported lease backend: ${raw ?? ''}`); +} + +export function normalizeDeviceKey(raw: string | undefined): string | undefined { + if (raw === undefined) return undefined; + const value = raw.trim(); + if (!value || value.length > 256 || !/^[\u0020-\u007E]+$/.test(value)) { + throw new AppError('INVALID_ARGS', 'Invalid device key. Use 1-256 printable characters.'); + } + return value; +} + +function normalizeClientId(raw: string | undefined): string | undefined { + return normalizeAgentIdentifier(raw, 'client id', 128); +} + +export function normalizeLeaseProvider(raw: string | undefined): string | undefined { + return normalizeAgentIdentifier(raw, 'lease provider', 64); +} + +function normalizeAgentIdentifier( + raw: string | undefined, + label: string, + maxLength: number, +): string | undefined { + if (raw === undefined) return undefined; + const value = raw.trim(); + if (!value || value.length > maxLength || !/^[a-zA-Z0-9._-]+$/.test(value)) { + throw new AppError( + 'INVALID_ARGS', + `Invalid ${label}. Use 1-${String(maxLength)} chars: letters, numbers, dot, underscore, hyphen.`, + ); + } + return value; +} + +function normalizeRequiredTenantId(raw: string): string { + const tenantId = normalizeTenantId(raw); + if (!tenantId) { + throw new AppError( + 'INVALID_ARGS', + 'Invalid tenant id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', + ); + } + return tenantId; +} + +function normalizeRequiredRunId(raw: string): string { + const runId = normalizeRunId(raw); + if (!runId) { + throw new AppError( + 'INVALID_ARGS', + 'Invalid run id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', + ); + } + return runId; +} + +export function normalizeAllocateLeaseRequest( + request: AllocateLeaseRequest, +): NormalizedAllocateLeaseRequest { + return { + backend: normalizeLeaseBackend(request.leaseBackend), + leaseProvider: normalizeLeaseProvider(request.leaseProvider), + deviceKey: normalizeDeviceKey(request.deviceKey), + clientId: normalizeClientId(request.clientId), + tenantId: normalizeRequiredTenantId(request.tenantId), + runId: normalizeRequiredRunId(request.runId), + ttlMs: request.ttlMs, + }; +} + +function leaseRequiresOwnerScope(lease: DeviceLease): boolean { + return Boolean(lease.leaseProvider ?? lease.deviceKey ?? lease.clientId); +} + +function hasRequiredOwnerScope(lease: DeviceLease, request: LeaseScopeMatchRequest): boolean { + if (!request.tenantId || !request.runId) return false; + return [ + [lease.leaseProvider, request.leaseProvider], + [lease.deviceKey, request.deviceKey], + [lease.clientId, request.clientId], + ].every(([leaseValue, requestValue]) => !leaseValue || Boolean(requestValue)); +} + +export function assertLeaseScopeMatch( + lease: DeviceLease, + request: HeartbeatLeaseRequest | AdmissionRequest, +): void { + const normalized = normalizeOptionalLeaseScope(request); + if ( + (normalized.tenantId && lease.tenantId !== normalized.tenantId) || + (normalized.runId && lease.runId !== normalized.runId) || + (normalized.leaseBackend && lease.backend !== normalized.leaseBackend) || + (normalized.leaseProvider && lease.leaseProvider !== normalized.leaseProvider) || + (normalized.deviceKey && lease.deviceKey !== normalized.deviceKey) || + (normalized.clientId && lease.clientId !== normalized.clientId) + ) { + throw new AppError('UNAUTHORIZED', 'Lease does not match tenant/run scope', { + reason: 'LEASE_SCOPE_MISMATCH', + }); + } +} + +export function normalizeLeaseAdmissionRequest( + request: AdmissionRequest, +): AdmissionRequest & { leaseId: string } { + const leaseBackend = normalizeLeaseBackend(request.leaseBackend); + const tenantId = normalizeTenantId(request.tenantId); + if (!tenantId) throw new AppError('INVALID_ARGS', 'tenant isolation requires tenant id.'); + const runId = normalizeRunId(request.runId); + if (!runId) throw new AppError('INVALID_ARGS', 'tenant isolation requires run id.'); + const leaseId = normalizeLeaseId(request.leaseId); + if (!leaseId) throw new AppError('INVALID_ARGS', 'tenant isolation requires lease id.'); + return { ...request, tenantId, runId, leaseId, leaseBackend }; +} +function normalizeOptionalLeaseScope( + request: LeaseScopeMatchRequest, +): NormalizedLeaseScopeMatchRequest { + const tenantId = normalizeTenantId(request.tenantId); + const runId = normalizeRunId(request.runId); + if (request.tenantId && !tenantId) { + throw new AppError( + 'INVALID_ARGS', + 'Invalid tenant id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', + ); + } + if (request.runId && !runId) { + throw new AppError( + 'INVALID_ARGS', + 'Invalid run id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', + ); + } + return { + tenantId, + runId, + leaseBackend: request.leaseBackend ? normalizeLeaseBackend(request.leaseBackend) : undefined, + leaseProvider: normalizeLeaseProvider(request.leaseProvider), + deviceKey: normalizeDeviceKey(request.deviceKey), + clientId: normalizeClientId(request.clientId), + }; +} + +export function assertLeaseOwnerScope(lease: DeviceLease, request: HeartbeatLeaseRequest): void { + if (leaseRequiresOwnerScope(lease) && !hasRequiredOwnerScope(lease, request)) { + throw new AppError('UNAUTHORIZED', 'Lease owner scope is required', { + reason: 'LEASE_SCOPE_REQUIRED', + }); + } +} + +export function leaseDeviceBindingKey( + scope: Pick, +): string | undefined { + if (!scope.deviceKey) return undefined; + return JSON.stringify([ + scope.backend, + scope.leaseProvider ?? DEFAULT_LEASE_PROVIDER, + scope.deviceKey, + ]); +} + +export function leaseRunBindingKey( + scope: Pick, +): string { + return JSON.stringify([ + scope.tenantId, + scope.runId, + scope.backend, + scope.leaseProvider ?? DEFAULT_LEASE_PROVIDER, + scope.deviceKey ?? '*', + ]); +} + +export function createDeviceLease( + request: NormalizedAllocateLeaseRequest, + leaseTtlMs: number, + now: number, +): DeviceLease { + return { + leaseId: crypto.randomBytes(16).toString('hex'), + tenantId: request.tenantId, + runId: request.runId, + backend: request.backend, + ...(request.leaseProvider ? { leaseProvider: request.leaseProvider } : {}), + ...(request.deviceKey ? { deviceKey: request.deviceKey } : {}), + ...(request.clientId ? { clientId: request.clientId } : {}), + createdAt: now, + heartbeatAt: now, + expiresAt: now + leaseTtlMs, + }; +} + +export function deviceLeaseBusyError(activeLease: DeviceLease): AppError { + return new AppError('DEVICE_IN_USE', 'Device is already leased', { + reason: 'DEVICE_LEASE_BUSY', + deviceKey: activeLease.deviceKey, + backend: activeLease.backend, + leaseProvider: activeLease.leaseProvider, + expiresAt: activeLease.expiresAt, + hint: 'Retry after the lease expires or close the owning session.', + }); +} diff --git a/src/daemon/lease-registry.ts b/src/daemon/lease-registry.ts index d25b345b6..35e9e8969 100644 --- a/src/daemon/lease-registry.ts +++ b/src/daemon/lease-registry.ts @@ -1,212 +1,52 @@ import type { DeviceLease } from '@agent-device/contracts/device'; -import crypto from 'node:crypto'; +import type { HumanControlHold, HumanControlHoldScope } from '@agent-device/contracts/client'; import type { LeaseBackend } from '@agent-device/kernel/contracts'; import { AppError } from '@agent-device/kernel/errors'; -import { normalizeTenantId } from './config.ts'; import { ProviderSessionOwnershipRegistry, type ProviderSessionOwnership, } from './provider-session-ownership.ts'; +import { + type AdmissionRequest, + type AllocateLeaseRequest, + type HeartbeatLeaseRequest, + type ReleaseLeaseRequest, + type LeaseRegistryOptions, + type NormalizedAllocateLeaseRequest, + createLeaseTtlResolver, + normalizeAllocateLeaseRequest, + createDeviceLease, + deviceLeaseBusyError, + normalizeLeaseId, + normalizeRequiredLeaseId, + normalizeLeaseAdmissionRequest, + assertLeaseOwnerScope, + assertLeaseScopeMatch, + leaseDeviceBindingKey, + leaseRunBindingKey, +} from './lease-registry-scope.ts'; +import { DeviceMutationDrain } from './device-mutation-drain.ts'; +import { + type HumanControlAuthority, + type HumanControlHoldInput, + normalizeHumanControlHoldId, + parseHumanControlHoldInput, + cloneHumanControlHold, + humanControlActiveError, +} from './human-control-contract.ts'; export type SimulatorLease = DeviceLease; -export type LeaseRegistryOptions = { - maxActiveSimulatorLeases?: number; - defaultLeaseTtlMs?: number; - minLeaseTtlMs?: number; - maxLeaseTtlMs?: number; - providerSessionRetentionMs?: number; - now?: () => number; - onLeaseExpired?: (lease: DeviceLease) => void; -}; - -export type AllocateLeaseRequest = { - tenantId: string; - runId: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; - ttlMs?: number; -}; - -export type HeartbeatLeaseRequest = { - leaseId: string; - tenantId?: string; - runId?: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; - ttlMs?: number; -}; - -export type ReleaseLeaseRequest = { - leaseId: string; - tenantId?: string; - runId?: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; -}; - -export type AdmissionRequest = { - tenantId?: string; - runId?: string; - leaseId?: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; -}; - -type LeaseScopeMatchRequest = { - tenantId?: string; - runId?: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; -}; - -type NormalizedLeaseScopeMatchRequest = { - tenantId?: string; - runId?: string; - leaseBackend?: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; -}; - -type NormalizedAllocateLeaseRequest = { - tenantId: string; - runId: string; - backend: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - clientId?: string; - ttlMs?: number; -}; - -const DEFAULT_LEASE_TTL_MS = 60_000; -const MIN_LEASE_TTL_MS = 5_000; -const MAX_LEASE_TTL_MS = 10 * 60_000; -const DEFAULT_LEASE_PROVIDER = 'default'; - -function normalizeRunId(raw: string | undefined): string | undefined { - if (!raw) return undefined; - const value = raw.trim(); - if (!value) return undefined; - if (!/^[a-zA-Z0-9._-]{1,128}$/.test(value)) return undefined; - return value; -} - -function normalizeLeaseId(raw: string | undefined): string | undefined { - if (!raw) return undefined; - const value = raw.trim(); - if (!value) return undefined; - if (!/^[a-f0-9]{16,128}$/i.test(value)) return undefined; - return value.toLowerCase(); -} - -function normalizeLeaseBackend(raw: string | undefined): LeaseBackend { - const value = (raw ?? '').trim().toLowerCase(); - if (!value || value === 'ios-simulator') return 'ios-simulator'; - if (value === 'ios-instance' || value === 'android-instance') return value; - throw new AppError('INVALID_ARGS', `Unsupported lease backend: ${raw ?? ''}`); -} - -function normalizeDeviceKey(raw: string | undefined): string | undefined { - if (raw === undefined) return undefined; - const value = raw.trim(); - if (!value || value.length > 256 || !/^[\u0020-\u007E]+$/.test(value)) { - throw new AppError('INVALID_ARGS', 'Invalid device key. Use 1-256 printable characters.'); - } - return value; -} - -function normalizeClientId(raw: string | undefined): string | undefined { - return normalizeAgentIdentifier(raw, 'client id', 128); -} - -function normalizeLeaseProvider(raw: string | undefined): string | undefined { - return normalizeAgentIdentifier(raw, 'lease provider', 64); -} - -function normalizeAgentIdentifier( - raw: string | undefined, - label: string, - maxLength: number, -): string | undefined { - if (raw === undefined) return undefined; - const value = raw.trim(); - if (!value || value.length > maxLength || !/^[a-zA-Z0-9._-]+$/.test(value)) { - throw new AppError( - 'INVALID_ARGS', - `Invalid ${label}. Use 1-${String(maxLength)} chars: letters, numbers, dot, underscore, hyphen.`, - ); - } - return value; -} - -function normalizeRequiredTenantId(raw: string): string { - const tenantId = normalizeTenantId(raw); - if (!tenantId) { - throw new AppError( - 'INVALID_ARGS', - 'Invalid tenant id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', - ); - } - return tenantId; -} - -function normalizeRequiredRunId(raw: string): string { - const runId = normalizeRunId(raw); - if (!runId) { - throw new AppError( - 'INVALID_ARGS', - 'Invalid run id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', - ); - } - return runId; -} - -function normalizeAllocateLeaseRequest( - request: AllocateLeaseRequest, -): NormalizedAllocateLeaseRequest { - return { - backend: normalizeLeaseBackend(request.leaseBackend), - leaseProvider: normalizeLeaseProvider(request.leaseProvider), - deviceKey: normalizeDeviceKey(request.deviceKey), - clientId: normalizeClientId(request.clientId), - tenantId: normalizeRequiredTenantId(request.tenantId), - runId: normalizeRequiredRunId(request.runId), - ttlMs: request.ttlMs, - }; -} - -function leaseRequiresOwnerScope(lease: DeviceLease): boolean { - return Boolean(lease.leaseProvider ?? lease.deviceKey ?? lease.clientId); -} - -function hasRequiredOwnerScope(lease: DeviceLease, request: LeaseScopeMatchRequest): boolean { - if (!request.tenantId || !request.runId) return false; - return [ - [lease.leaseProvider, request.leaseProvider], - [lease.deviceKey, request.deviceKey], - [lease.clientId, request.clientId], - ].every(([leaseValue, requestValue]) => !leaseValue || Boolean(requestValue)); -} +type OwnedHumanControlHold = { hold: HumanControlHold; ownerLeaseId?: string }; export class LeaseRegistry { + private readonly holdsByDevice = new Map>(); + private readonly mutations = new DeviceMutationDrain(); private readonly leases = new Map(); private readonly runBindings = new Map(); private readonly deviceBindings = new Map(); private readonly maxActiveSimulatorLeases: number; - private readonly defaultLeaseTtlMs: number; - private readonly minLeaseTtlMs: number; - private readonly maxLeaseTtlMs: number; + private readonly resolveLeaseTtlMs: ReturnType; private readonly now: () => number; private readonly onLeaseExpired?: (lease: DeviceLease) => void; private readonly providerSessionOwnership: ProviderSessionOwnershipRegistry; @@ -215,15 +55,7 @@ export class LeaseRegistry { this.maxActiveSimulatorLeases = Number.isInteger(options.maxActiveSimulatorLeases) ? Math.max(0, Number(options.maxActiveSimulatorLeases)) : 0; - this.defaultLeaseTtlMs = Number.isInteger(options.defaultLeaseTtlMs) - ? Math.max(1, Number(options.defaultLeaseTtlMs)) - : DEFAULT_LEASE_TTL_MS; - this.minLeaseTtlMs = Number.isInteger(options.minLeaseTtlMs) - ? Math.max(1, Number(options.minLeaseTtlMs)) - : MIN_LEASE_TTL_MS; - this.maxLeaseTtlMs = Number.isInteger(options.maxLeaseTtlMs) - ? Math.max(this.minLeaseTtlMs, Number(options.maxLeaseTtlMs)) - : MAX_LEASE_TTL_MS; + this.resolveLeaseTtlMs = createLeaseTtlResolver(options); this.now = options.now ?? (() => Date.now()); this.onLeaseExpired = options.onLeaseExpired; this.providerSessionOwnership = new ProviderSessionOwnershipRegistry({ @@ -236,11 +68,12 @@ export class LeaseRegistry { const normalized = normalizeAllocateLeaseRequest(request); this.cleanupExpiredLeases(); const leaseTtlMs = this.resolveLeaseTtlMs(normalized.ttlMs); + this.assertHumanControlAdmission(normalized); const existingLease = this.refreshExistingRunBinding(normalized, leaseTtlMs); if (existingLease) return existingLease; this.assertDeviceAvailable(normalized); this.enforceCapacity(normalized.backend); - const lease = this.createLease(normalized, leaseTtlMs); + const lease = createDeviceLease(normalized, leaseTtlMs, this.now()); this.leases.set(lease.leaseId, lease); this.bindLease(lease); return { ...lease }; @@ -250,7 +83,7 @@ export class LeaseRegistry { request: NormalizedAllocateLeaseRequest, leaseTtlMs: number, ): DeviceLease | undefined { - const bindingKey = this.bindingKey(request); + const bindingKey = leaseRunBindingKey(request); const existingId = this.runBindings.get(bindingKey); if (!existingId) return undefined; const existingLease = this.leases.get(existingId); @@ -258,38 +91,22 @@ export class LeaseRegistry { this.runBindings.delete(bindingKey); return undefined; } - if (this.canReuseRunBinding(existingLease, request)) { + if (existingLease.clientId === request.clientId) { return this.refreshLease(existingLease, leaseTtlMs); } if (existingLease.deviceKey) { - this.throwDeviceBusy(existingLease); + throw deviceLeaseBusyError(existingLease); } - this.assertOptionalLeaseIdentityMatch(existingLease, request); + assertLeaseScopeMatch(existingLease, request); return this.refreshLease(existingLease, leaseTtlMs); } - private createLease(request: NormalizedAllocateLeaseRequest, leaseTtlMs: number): DeviceLease { - const now = this.now(); - return { - leaseId: crypto.randomBytes(16).toString('hex'), - tenantId: request.tenantId, - runId: request.runId, - backend: request.backend, - ...(request.leaseProvider ? { leaseProvider: request.leaseProvider } : {}), - ...(request.deviceKey ? { deviceKey: request.deviceKey } : {}), - ...(request.clientId ? { clientId: request.clientId } : {}), - createdAt: now, - heartbeatAt: now, - expiresAt: now + leaseTtlMs, - }; - } - heartbeatLease(request: HeartbeatLeaseRequest): DeviceLease { - const leaseId = this.normalizeRequiredLeaseId(request.leaseId); + const leaseId = normalizeRequiredLeaseId(request.leaseId); this.cleanupExpiredLeases(); const lease = this.getActiveLease(leaseId); - this.assertRequiredScopeForDeviceAwareLease(lease, request); - this.assertOptionalScopeMatch(lease, request); + assertLeaseOwnerScope(lease, request); + assertLeaseScopeMatch(lease, request); const leaseTtlMs = this.resolveLeaseTtlMs(request.ttlMs); return this.refreshLease(lease, leaseTtlMs); } @@ -310,39 +127,19 @@ export class LeaseRegistry { * transient provider failure leaves the local lease available for retry. */ getLease(request: ReleaseLeaseRequest): DeviceLease | undefined { - const leaseId = this.normalizeRequiredLeaseId(request.leaseId); + const leaseId = normalizeRequiredLeaseId(request.leaseId); this.cleanupExpiredLeases(); const lease = this.leases.get(leaseId); if (!lease) return undefined; - this.assertRequiredScopeForDeviceAwareLease(lease, request); - this.assertOptionalScopeMatch(lease, request); + assertLeaseOwnerScope(lease, request); + assertLeaseScopeMatch(lease, request); return { ...lease }; } assertLeaseAdmission(request: AdmissionRequest): void { - const backend = normalizeLeaseBackend(request.leaseBackend); - const tenantId = normalizeTenantId(request.tenantId); - if (!tenantId) { - throw new AppError('INVALID_ARGS', 'tenant isolation requires tenant id.'); - } - const runId = normalizeRunId(request.runId); - if (!runId) { - throw new AppError('INVALID_ARGS', 'tenant isolation requires run id.'); - } - const leaseId = normalizeLeaseId(request.leaseId); - if (!leaseId) { - throw new AppError('INVALID_ARGS', 'tenant isolation requires lease id.'); - } + const scope = normalizeLeaseAdmissionRequest(request); this.cleanupExpiredLeases(); - const lease = this.getActiveLease(leaseId); - this.assertOptionalScopeMatch(lease, { - tenantId, - runId, - leaseBackend: backend, - leaseProvider: request.leaseProvider, - deviceKey: request.deviceKey, - clientId: request.clientId, - }); + assertLeaseScopeMatch(this.getActiveLease(scope.leaseId), scope); } listActiveLeases(): DeviceLease[] { @@ -371,11 +168,231 @@ export class LeaseRegistry { return this.providerSessionOwnership.resolve(params); } + listHumanControlHolds(authority: HumanControlAuthority): HumanControlHold[] { + this.cleanupExpiredLeases(); + const key = + authority.kind === 'lease' + ? leaseDeviceBindingKey(this.requireHumanControlLease(authority.leaseId)) + : undefined; + const holds = [...this.holdsByDevice.entries()] + .filter(([deviceKey]) => key === undefined || key === deviceKey) + .flatMap(([, entries]) => + [...entries.values()].map(({ hold }) => cloneHumanControlHold(hold)), + ); + return holds.sort((left, right) => left.id.localeCompare(right.id)); + } + + async putHumanControlHold( + authority: HumanControlAuthority, + rawId: string, + rawInput: HumanControlHoldInput, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted(); + const id = normalizeHumanControlHoldId(rawId); + const input = parseHumanControlHoldInput(rawInput); + this.cleanupExpiredLeases(); + const scope = this.resolveHumanControlScope(authority, input); + const key = leaseDeviceBindingKey(scope)!; + const existing = this.findHumanControlHold(id); + if (existing) { + this.assertHoldAuthority(authority, existing); + if (leaseDeviceBindingKey(existing.hold.scope) !== key) { + throw new AppError('INVALID_ARGS', 'Release a hold before changing its device scope.'); + } + } + const now = this.now(); + const pending: OwnedHumanControlHold = { + ...(authority.kind === 'lease' ? { ownerLeaseId: authority.leaseId } : {}), + hold: { + id, + scope, + state: 'activating', + ...(input.reason ? { reason: input.reason } : {}), + createdAt: existing?.hold.createdAt ?? now, + updatedAt: now, + }, + }; + const holds = this.holdsByDevice.get(key) ?? new Map(); + this.holdsByDevice.set(key, holds); + holds.set(id, pending); + return await this.activateHumanControlHold(key, pending, input.ttlMs, signal); + } + + private async activateHumanControlHold( + key: string, + pending: OwnedHumanControlHold, + ttlMs: number | undefined, + signal?: AbortSignal, + ): Promise { + const holds = this.holdsByDevice.get(key)!; + const { id } = pending.hold; + try { + await this.mutations.wait(key, signal); + signal?.throwIfAborted(); + if (holds.get(id) !== pending) { + throw new AppError( + 'COMMAND_FAILED', + 'Human-control hold changed before activation completed.', + { holdId: id }, + ); + } + const activatedAt = this.now(); + pending.hold = { + ...pending.hold, + state: 'active', + updatedAt: activatedAt, + ...(ttlMs === undefined ? {} : { expiresAt: activatedAt + ttlMs }), + }; + this.refreshHeldLease(key, activatedAt); + return cloneHumanControlHold(pending.hold); + } catch (error) { + if (holds.get(id) === pending) this.deleteHumanControlHold(key, id); + throw error; + } + } + + removeHumanControlHold( + authority: HumanControlAuthority, + rawId: string, + ): HumanControlHold | undefined { + const id = normalizeHumanControlHoldId(rawId); + this.cleanupExpiredLeases(); + const existing = this.findHumanControlHold(id); + if (!existing) return undefined; + this.assertHoldAuthority(authority, existing); + const key = leaseDeviceBindingKey(existing.hold.scope)!; + this.deleteHumanControlHold(key, id); + return cloneHumanControlHold(existing.hold); + } + + private deleteHumanControlHold(key: string, id: string): void { + const holds = this.holdsByDevice.get(key)!; + holds.delete(id); + if (holds.size === 0) { + this.holdsByDevice.delete(key); + this.refreshHeldLease(key, this.now()); + } + } + + assertHumanControlAdmission( + scope: Pick, + ): void { + this.expireHumanControlHolds(); + const key = leaseDeviceBindingKey(scope); + const entry = + key === undefined ? undefined : this.holdsByDevice.get(key)?.values().next().value; + if (entry) throw humanControlActiveError(entry.hold); + } + + async runDeviceMutation(lease: DeviceLease | undefined, task: () => Promise): Promise { + if (!lease) return await task(); + this.cleanupExpiredLeases(); + const activeLease = this.getActiveLease(lease.leaseId); + this.assertHumanControlAdmission(activeLease); + const key = leaseDeviceBindingKey(activeLease); + return key === undefined ? await task() : await this.mutations.run(key, task); + } + + private requireHumanControlLease(leaseId: string): DeviceLease & { deviceKey: string } { + const lease = this.getActiveLease(leaseId); + if (!lease.deviceKey) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Human control requires a device-scoped remote lease.', + ); + } + return { ...lease, deviceKey: lease.deviceKey }; + } + + private resolveHumanControlScope( + authority: HumanControlAuthority, + input: HumanControlHoldInput, + ): HumanControlHoldScope { + if (authority.kind === 'host') { + if (!input.scope) + throw new AppError('INVALID_ARGS', 'Host human control requires a lease device scope.'); + return input.scope; + } + if (input.scope) { + throw new AppError( + 'INVALID_ARGS', + 'Lease-owner human control uses the admitted lease device; do not supply scope.', + ); + } + const lease = this.requireHumanControlLease(authority.leaseId); + return { + backend: lease.backend, + leaseProvider: lease.leaseProvider, + deviceKey: lease.deviceKey, + }; + } + + private assertHoldAuthority( + authority: HumanControlAuthority, + entry: OwnedHumanControlHold, + ): void { + if (authority.kind === 'host') return; + const lease = this.requireHumanControlLease(authority.leaseId); + if ( + entry.ownerLeaseId !== lease.leaseId || + leaseDeviceBindingKey(lease) !== leaseDeviceBindingKey(entry.hold.scope) + ) { + throw new AppError( + 'UNAUTHORIZED', + 'Human-control hold does not belong to the admitted lease.', + { reason: 'LEASE_SCOPE_MISMATCH' }, + ); + } + } + + private findHumanControlHold(id: string): OwnedHumanControlHold | undefined { + for (const holds of this.holdsByDevice.values()) { + const entry = holds.get(id); + if (entry) return entry; + } + return undefined; + } + + private hasHumanControl(lease: DeviceLease): boolean { + const key = leaseDeviceBindingKey(lease); + return key !== undefined && this.holdsByDevice.has(key); + } + + private expireHumanControlHolds(): void { + const now = this.now(); + for (const [key, holds] of this.holdsByDevice) { + let releasedAt = 0; + for (const [id, { hold }] of holds) { + if (hold.expiresAt === undefined || hold.expiresAt > now) continue; + releasedAt = Math.max(releasedAt, hold.expiresAt); + holds.delete(id); + } + if (holds.size === 0) { + this.holdsByDevice.delete(key); + this.refreshHeldLease(key, releasedAt); + } + } + } + + private refreshHeldLease(key: string, at: number): void { + const leaseId = this.deviceBindings.get(key); + const lease = leaseId ? this.leases.get(leaseId) : undefined; + if (lease) { + this.refreshLease( + lease, + lease.expiresAt - lease.heartbeatAt, + Math.max(at, lease.heartbeatAt), + ); + } + } + consumeExpiredLeases(): DeviceLease[] { + this.expireHumanControlHolds(); const now = this.now(); const expired: DeviceLease[] = []; for (const lease of this.leases.values()) { - if (lease.expiresAt > now) continue; + if (lease.expiresAt > now || this.hasHumanControl(lease)) continue; this.leases.delete(lease.leaseId); this.unbindLease(lease, lease.expiresAt); const expiredLease = { ...lease }; @@ -386,10 +403,13 @@ export class LeaseRegistry { } consumeExpiredLease(leaseId: string): DeviceLease | undefined { + this.expireHumanControlHolds(); const normalizedLeaseId = normalizeLeaseId(leaseId); if (!normalizedLeaseId) return undefined; const lease = this.leases.get(normalizedLeaseId); - if (!lease || lease.expiresAt > this.now()) return undefined; + if (!lease || lease.expiresAt > this.now() || this.hasHumanControl(lease)) { + return undefined; + } this.leases.delete(lease.leaseId); this.unbindLease(lease, lease.expiresAt); const expiredLease = { ...lease }; @@ -417,26 +437,6 @@ export class LeaseRegistry { }); } - private resolveLeaseTtlMs(raw: number | undefined): number { - if (!Number.isInteger(raw)) return this.defaultLeaseTtlMs; - const value = Number(raw); - if (value < this.minLeaseTtlMs || value > this.maxLeaseTtlMs) { - throw new AppError( - 'INVALID_ARGS', - `Lease ttlMs must be between ${this.minLeaseTtlMs} and ${this.maxLeaseTtlMs}.`, - ); - } - return value; - } - - private normalizeRequiredLeaseId(raw: string | undefined): string { - const leaseId = normalizeLeaseId(raw); - if (!leaseId) { - throw new AppError('INVALID_ARGS', 'Invalid lease id.'); - } - return leaseId; - } - private getActiveLease(leaseId: string): DeviceLease { const lease = this.leases.get(leaseId); if (lease) return lease; @@ -445,8 +445,7 @@ export class LeaseRegistry { }); } - private refreshLease(lease: DeviceLease, ttlMs: number): DeviceLease { - const now = this.now(); + private refreshLease(lease: DeviceLease, ttlMs: number, now = this.now()): DeviceLease { const updated: DeviceLease = { ...lease, heartbeatAt: now, @@ -458,72 +457,28 @@ export class LeaseRegistry { } private bindLease(lease: DeviceLease): void { - this.runBindings.set( - this.bindingKey({ - tenantId: lease.tenantId, - runId: lease.runId, - backend: lease.backend, - leaseProvider: lease.leaseProvider, - deviceKey: lease.deviceKey, - }), - lease.leaseId, - ); - const deviceBindingKey = this.deviceBindingKey(lease); + this.runBindings.set(leaseRunBindingKey(lease), lease.leaseId); + const deviceBindingKey = leaseDeviceBindingKey(lease); if (deviceBindingKey) { this.deviceBindings.set(deviceBindingKey, lease.leaseId); } } private unbindLease(lease: DeviceLease, releasedAt = this.now()): void { - this.runBindings.delete( - this.bindingKey({ - tenantId: lease.tenantId, - runId: lease.runId, - backend: lease.backend, - leaseProvider: lease.leaseProvider, - deviceKey: lease.deviceKey, - }), - ); - const deviceBindingKey = this.deviceBindingKey(lease); + this.runBindings.delete(leaseRunBindingKey(lease)); + const deviceBindingKey = leaseDeviceBindingKey(lease); if (deviceBindingKey) { this.deviceBindings.delete(deviceBindingKey); } this.providerSessionOwnership.markLeaseReleased(lease, releasedAt); } - private bindingKey(params: { - tenantId: string; - runId: string; - backend: LeaseBackend; - leaseProvider?: string; - deviceKey?: string; - }): string { - return JSON.stringify([ - params.tenantId, - params.runId, - params.backend, - params.leaseProvider ?? DEFAULT_LEASE_PROVIDER, - params.deviceKey ?? '*', - ]); - } - - private deviceBindingKey( - lease: Pick, - ): string | undefined { - if (!lease.deviceKey) return undefined; - return JSON.stringify([ - lease.backend, - lease.leaseProvider ?? DEFAULT_LEASE_PROVIDER, - lease.deviceKey, - ]); - } - private assertDeviceAvailable(params: { backend: LeaseBackend; leaseProvider?: string; deviceKey?: string; }): void { - const deviceBindingKey = this.deviceBindingKey({ + const deviceBindingKey = leaseDeviceBindingKey({ backend: params.backend, leaseProvider: params.leaseProvider, deviceKey: params.deviceKey, @@ -536,106 +491,6 @@ export class LeaseRegistry { this.deviceBindings.delete(deviceBindingKey); return; } - this.throwDeviceBusy(activeLease); - } - - private canReuseRunBinding( - lease: DeviceLease, - request: { - clientId?: string; - }, - ): boolean { - return lease.clientId === request.clientId; - } - - private throwDeviceBusy(activeLease: DeviceLease): never { - throw new AppError('DEVICE_IN_USE', 'Device is already leased', { - reason: 'DEVICE_LEASE_BUSY', - deviceKey: activeLease.deviceKey, - backend: activeLease.backend, - leaseProvider: activeLease.leaseProvider, - expiresAt: activeLease.expiresAt, - hint: 'Retry after the lease expires or close the owning session.', - }); - } - - private assertRequiredScopeForDeviceAwareLease( - lease: DeviceLease, - request: LeaseScopeMatchRequest, - ): void { - if (!leaseRequiresOwnerScope(lease)) return; - if (!hasRequiredOwnerScope(lease, request)) { - this.throwScopeRequired(); - } - } - - private assertOptionalScopeMatch(lease: DeviceLease, request: LeaseScopeMatchRequest): void { - const normalized = this.normalizeOptionalScopeMatchRequest(request); - if ( - (normalized.tenantId && lease.tenantId !== normalized.tenantId) || - (normalized.runId && lease.runId !== normalized.runId) || - (normalized.leaseBackend && lease.backend !== normalized.leaseBackend) - ) { - this.throwScopeMismatch(); - } - this.assertOptionalLeaseIdentityMatch(lease, normalized); - } - - private normalizeOptionalScopeMatchRequest( - request: LeaseScopeMatchRequest, - ): NormalizedLeaseScopeMatchRequest { - const tenantId = normalizeTenantId(request.tenantId); - const runId = normalizeRunId(request.runId); - if (request.tenantId && !tenantId) { - throw new AppError( - 'INVALID_ARGS', - 'Invalid tenant id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', - ); - } - if (request.runId && !runId) { - throw new AppError( - 'INVALID_ARGS', - 'Invalid run id. Use 1-128 chars: letters, numbers, dot, underscore, hyphen.', - ); - } - return { - tenantId, - runId, - leaseBackend: request.leaseBackend ? normalizeLeaseBackend(request.leaseBackend) : undefined, - leaseProvider: normalizeLeaseProvider(request.leaseProvider), - deviceKey: normalizeDeviceKey(request.deviceKey), - clientId: normalizeClientId(request.clientId), - }; - } - - private assertOptionalLeaseIdentityMatch( - lease: DeviceLease, - request: { - leaseProvider?: string; - deviceKey?: string; - clientId?: string; - }, - ): void { - if (request.leaseProvider && lease.leaseProvider !== request.leaseProvider) { - this.throwScopeMismatch(); - } - if (request.deviceKey && lease.deviceKey !== request.deviceKey) { - this.throwScopeMismatch(); - } - if (request.clientId && lease.clientId !== request.clientId) { - this.throwScopeMismatch(); - } - } - - private throwScopeMismatch(): never { - throw new AppError('UNAUTHORIZED', 'Lease does not match tenant/run scope', { - reason: 'LEASE_SCOPE_MISMATCH', - }); - } - - private throwScopeRequired(): never { - throw new AppError('UNAUTHORIZED', 'Lease owner scope is required', { - reason: 'LEASE_SCOPE_REQUIRED', - }); + throw deviceLeaseBusyError(activeLease); } } diff --git a/src/daemon/request-admission.ts b/src/daemon/request-admission.ts index 655350751..2f5628cd1 100644 --- a/src/daemon/request-admission.ts +++ b/src/daemon/request-admission.ts @@ -3,6 +3,7 @@ import { normalizeTenantId, resolveSessionIsolationMode } from './config.ts'; import { isTenantOwnedSessionName, tenantScopedSessionName } from './session-tenant-scope.ts'; import { isLeaseAdmissionExempt, + isHumanControlMutation, isSessionlessPlainCloseAdmissionExempt, } from './daemon-command-registry.ts'; import { @@ -91,7 +92,13 @@ export function assertRequestLeaseAdmission( ) { return undefined; } - if (!sessionLease && req.meta?.sessionIsolation !== 'tenant') { + if (req.command === 'human_control' && !sessionLease && !requestLeaseScope.leaseId) { + throw new AppError( + 'UNSUPPORTED_OPERATION', + 'Takeover requires an active remote device lease. Local takeover is not supported.', + ); + } + if (req.command !== 'human_control' && !sessionLease && req.meta?.sessionIsolation !== 'tenant') { if (!requestLeaseScope.leaseId) return undefined; if (!requestLeaseScope.tenantId && !requestLeaseScope.runId) return undefined; } @@ -104,7 +111,9 @@ export function assertRequestLeaseAdmission( (isProxyLeaseScope(leaseScope) ? DEFAULT_PROXY_LEASE_TTL_MS : undefined), }; leaseRegistry.assertLeaseAdmission(leaseScopeToHeartbeatRequest(leaseScope)); - return leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(heartbeatLeaseScope)); + const lease = leaseRegistry.heartbeatLease(leaseScopeToHeartbeatRequest(heartbeatLeaseScope)); + if (isHumanControlMutation(req)) leaseRegistry.assertHumanControlAdmission(lease); + return lease; } export function assertRequestLeaseAdmissionPreflight(req: DaemonRequest): void { diff --git a/src/daemon/request-execution-scope.ts b/src/daemon/request-execution-scope.ts index b66fe746e..da9fd447a 100644 --- a/src/daemon/request-execution-scope.ts +++ b/src/daemon/request-execution-scope.ts @@ -25,6 +25,7 @@ import { throwIfRequestCanceled } from '@agent-device/host-kit/request'; import { finalizeDaemonResponse } from './request-finalization.ts'; import { refreshRecordingHealth } from './request-recording-health.ts'; import { + isHumanControlMutation, shouldBlockForInvalidRecording, shouldLockSessionExecution, shouldValidateSessionSelector, @@ -237,7 +238,9 @@ export async function createRequestExecutionScope(params: { leaseRegistry, }); scope.req = scopedReq; - return await task(); + return isHumanControlMutation(scopedReq) + ? await leaseRegistry.runDeviceMutation(scopedReq.internal?.admittedLease, task) + : await task(); }, runLocked: async (task) => { throwIfRequestCanceled(scopedReq.meta?.requestId); diff --git a/src/daemon/request-handler-chain.ts b/src/daemon/request-handler-chain.ts index 3cb39940a..4ef68b81d 100644 --- a/src/daemon/request-handler-chain.ts +++ b/src/daemon/request-handler-chain.ts @@ -66,6 +66,10 @@ type RequestHandlerChainParams = { }; const DAEMON_ROUTE_HANDLERS = { + humanControl: defineDaemonRoute({ + load: () => import('./handlers/human-control.ts'), + run: runHumanControlHandler, + }), lease: defineDaemonRoute({ load: () => import('./handlers/lease.ts'), run: runLeaseHandler, @@ -121,6 +125,16 @@ export async function loadGenericRequestHandlerModule(): Promise< return await DAEMON_ROUTE_HANDLERS.generic.loadModule(); } +async function runHumanControlHandler( + { handleHumanControlCommand }: typeof import('./handlers/human-control.ts'), + params: RequestHandlerChainParams, +): Promise { + return await handleHumanControlCommand({ + req: params.req, + registry: params.leaseRegistry, + }); +} + async function runLeaseHandler( { handleLeaseCommands }: typeof import('./handlers/lease.ts'), params: RequestHandlerChainParams, diff --git a/src/daemon/route-owner-files.ts b/src/daemon/route-owner-files.ts index bd39a3bc2..15923d522 100644 --- a/src/daemon/route-owner-files.ts +++ b/src/daemon/route-owner-files.ts @@ -17,6 +17,7 @@ import type { DaemonCommandRoute } from './request-handler-chain.ts'; * each path still points at the module that route's loader imports. */ const DAEMON_ROUTE_OWNER_FILES = { + humanControl: 'src/daemon/handlers/human-control.ts', lease: 'src/daemon/handlers/lease.ts', session: 'src/daemon/handlers/session.ts', snapshot: 'src/daemon/handlers/snapshot.ts', diff --git a/src/daemon/server/daemon-runtime.ts b/src/daemon/server/daemon-runtime.ts index 82b838a89..1215c3406 100644 --- a/src/daemon/server/daemon-runtime.ts +++ b/src/daemon/server/daemon-runtime.ts @@ -444,6 +444,7 @@ export async function startDaemonRuntime( if (startHttpServer) { const httpServer = await createDaemonHttpServer({ handleRequest, + leaseRegistry, token, retainArtifacts, env, diff --git a/src/daemon/server/http-server.ts b/src/daemon/server/http-server.ts index 52b3317c4..f023417c6 100644 --- a/src/daemon/server/http-server.ts +++ b/src/daemon/server/http-server.ts @@ -43,6 +43,8 @@ import { tryHandleUploadHttpRoute } from '../upload-http.ts'; import { tryHandleDownloadableArtifactHttpRoute } from '../downloadable-artifact-http.ts'; import { tryHandleRequestDiagnosticsHttpRoute } from '../request-diagnostics-http.ts'; import { resolveTrustedTenant, tenantTrustRejectionError } from './tenant-trust.ts'; +import { tryHandleHumanControlHttpRoute } from '../human-control-http.ts'; +import type { LeaseRegistry } from '../lease-registry.ts'; type JsonRpcRequest = JsonRpcRequestEnvelope; @@ -552,6 +554,7 @@ async function loadHttpAuthHook( export async function createDaemonHttpServer(options: { handleRequest: DaemonInvokeFn; + leaseRegistry?: LeaseRegistry; token?: string; retainArtifacts?: boolean; env?: NodeJS.ProcessEnv; @@ -574,6 +577,19 @@ export async function createDaemonHttpServer(options: { return; } + if ( + token && + options.leaseRegistry && + tryHandleHumanControlHttpRoute({ + req, + res, + expectedToken: token, + registry: options.leaseRegistry, + }) + ) { + return; + } + if ( tryHandleUploadHttpRoute({ req, @@ -813,7 +829,7 @@ export async function createDaemonHttpServer(options: { daemonResponse.error.message, daemonResponse.error, ), - statusCodeForNormalizedError(daemonResponse.error.code), + statusCodeForDaemonError(daemonResponse.error), ); } catch (error) { handlerCompleted = true; @@ -838,6 +854,16 @@ export async function createDaemonHttpServer(options: { }); } +function statusCodeForDaemonError(error: { + code: string; + details?: Record; +}): number { + if (error.code === 'DEVICE_IN_USE' && error.details?.reason === 'human_control_active') { + return 423; + } + return statusCodeForNormalizedError(error.code); +} + async function authorizeAuxiliaryHttpRequest(params: { req: http.IncomingMessage; res: http.ServerResponse; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 454b6f804..5634f1098 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -108,6 +108,34 @@ agent-device open com.example.myapp --platform android --serial emulator-5554 -- agent-device metro reload ``` +## Human Takeover + +Use `takeover` with an active remote connection when a person needs to interact with its leased +device without racing the agent: + +```bash +agent-device takeover --session remote-session +agent-device takeover status --session remote-session +agent-device takeover release --session remote-session +``` + +The command uses the device from the admitted remote lease, installs a short-lived hold, keeps it +alive in the foreground, and releases it on Ctrl+C. Activation waits for admitted mutations to finish +before reporting active. While held, state-changing commands fail with +`DEVICE_IN_USE` and `details.reason: "human_control_active"`, explaining that agent interactions are +temporarily disabled. Snapshots, screenshots, selector reads, logs, and other read-only diagnostics +remain available. The hold also +protects an existing remote device lease from inactivity expiry so the human does not accidentally +hand the simulator to a different agent. + +A foreground hold expires automatically if its process disappears. Releasing or expiring the final +hold refreshes the existing lease's inactivity window. Tenant commands can modify only holds owned +by their admitted lease, not provider-host administrative holds. + +Holds do not survive daemon restart; reconnect and re-establish them before continuing human +interaction. Local takeover without a device-scoped remote lease is not supported in this version. +See [remote takeover and host administration](./remote-proxy.md#human-takeover) for the VM-side API. + ## Web Automation Minimal `--platform web` support reuses [agent-browser](https://github.com/vercel-labs/agent-browser). `agent-device` owns command/session/replay integration, refs/selectors, and artifact routing; `agent-browser` owns browser launch, page control, screenshots, and browser-specific mechanics. diff --git a/website/docs/docs/remote-proxy.md b/website/docs/docs/remote-proxy.md index bf97512b6..593a2a8ae 100644 --- a/website/docs/docs/remote-proxy.md +++ b/website/docs/docs/remote-proxy.md @@ -53,12 +53,74 @@ Do not put proxy endpoint, token, tenant, or provider fields in `./agent-device. configuration is intentionally limited to project-safe automation defaults. Use `connect proxy`, user config, an explicit `--config` file, or protected CI environment variables for the endpoint and token. +## Human Takeover + +With a remote device already leased by `open`, pause mutations through the same connection: + +```bash +agent-device takeover --session remote-session +``` + +The foreground command renews its hold until Ctrl+C. Read-only diagnostics remain available, the +agent session stays open, and its lease is protected from inactivity expiry. Activation waits for +already-admitted mutations to finish. Status and recovery use `takeover status` and +`takeover release ` with the same session. + +If the requesting connection disconnects while activation is waiting for mutations to finish, its +pending hold is removed and cannot activate later. This applies to both tenant RPCs and host PUTs. +Once active, holds follow their configured TTL or explicit release lifecycle. + +Lease-owner operations use ordinary `agent_device.command` RPCs at `POST /rpc`, with command +`human_control` and positionals `["list"]`, `["put", "", "{\"ttlMs\":15000}"]`, or +`["remove", ""]`. Supply the same tenant, run, client, lease, backend, provider, and device +metadata as other requests. The PUT payload contains only `reason` and `ttlMs`; the server derives +the target from the admitted lease. It rejects caller-supplied `scope`. + +### Host administration + +VM-side automation can manage holds independently of a tenant. Read the daemon's `httpPort` and +`token` from `daemon.json` in its effective state directory, then use the loopback listener with +`Authorization: Bearer ` or `X-Agent-Device-Token: `. An HTTP listener +is required. A tenant credential does not grant this capability. + +```text +PUT /admin/human-control/holds/ +GET /admin/human-control/holds +DELETE /admin/human-control/holds/ +``` + +The host PUT body names the exact lease contention identity, including its backend and provider. +Use the lease's `deviceKey`, not a bare device ID or a display name: + +```json +{ + "scope": { + "backend": "ios-instance", + "leaseProvider": "proxy", + "deviceKey": "ios:mobile:" + }, + "reason": "Human is using the VM console.", + "ttlMs": 15000 +} +``` + +Repeated PUT renews the hold. Omitting `ttlMs` keeps it until explicit release or daemon shutdown. +Tenant RPCs cannot modify host holds. Multiple holds can coexist; mutations resume only when all +holds on the device end. + +Holds do not survive daemon restart, matching lease state. Reconnect and re-establish the hold +before continuing human interaction. Local takeover without a device-scoped remote lease is +deferred; this does not provide a host-global fence across local daemons. + ## What Is Exposed The proxy allows only the daemon HTTP contract: `/health`, `/rpc`, `/upload` plus resumable `/upload/*` routes, and `/artifacts/*`, with the same routes also available under `/agent-device/*`. Health checks are unauthenticated; command, upload, and artifact routes require the bearer token. The proxy validates the client token and rewrites authorized upstream requests to the local daemon token. The local daemon still validates its own token, so the daemon token is not exposed to remote clients. +The proxy deliberately does not forward `/admin/*`, including human-control holds. A caller inside +the device-host VM must use the daemon's loopback port and local daemon token. + ## Compatibility Remote clients read `/health` before issuing commands and compare the daemon RPC protocol version. Keep the client and proxy versions reasonably close; patch-level differences should normally work, but incompatible RPC protocol versions fail before commands run. diff --git a/website/docs/docs/security-trust.md b/website/docs/docs/security-trust.md index 5df52dcef..a74abc7ba 100644 --- a/website/docs/docs/security-trust.md +++ b/website/docs/docs/security-trust.md @@ -18,11 +18,17 @@ description: Security and trust guidance for agent-device local app automation, CLI commands run through a per-user background daemon: - The daemon binds to `127.0.0.1` only, on ephemeral ports, for both its socket and HTTP transports. It is never reachable from the network unless you deliberately front it with your own proxy. -- Command (RPC), upload, and artifact-download requests must present a token generated fresh on each daemon boot (24 random bytes). The only unauthenticated endpoint is `GET /health`, which intentionally returns a bare liveness response and nothing else; like the rest of the server it is reachable only via loopback. The token is stored in `daemon.json` inside the daemon state directory (`~/.agent-device` for packaged installs; source checkouts use a worktree-scoped directory under `~/.agent-device/dev/`) with `0600` permissions; whoever can read that file already has your user account. +- Command (RPC), upload, artifact-download, and `/admin/human-control/*` requests must present a token generated fresh on each daemon boot (24 random bytes). The only unauthenticated endpoint is `GET /health`, which intentionally returns a bare liveness response and nothing else; like the rest of the server it is reachable only via loopback. The token is stored in `daemon.json` inside the daemon state directory (`~/.agent-device` for packaged installs; source checkouts use a worktree-scoped directory under `~/.agent-device/dev/`) with `0600` permissions; whoever can read that file already has your user account. - A client only reuses a running daemon when the daemon's version and binary code signature match its own; otherwise the daemon is restarted. This prevents a stale or tampered daemon from silently serving new clients. - Artifact uploads are size-capped, filenames are sanitized, and archive extraction rejects path-traversal entries. Artifact downloads resolve through server-side IDs, never client-supplied paths. -For remote or cloud deployments, the daemon supports a custom auth hook: `AGENT_DEVICE_HTTP_AUTH_HOOK` names a module path that is dynamically imported and invoked for each HTTP request (with `AGENT_DEVICE_HTTP_AUTH_EXPORT` selecting the export). The hook runs with the daemon's full privileges, so treat it as part of your trusted computing base: point it only at a read-only path you control, never at a location writable by less-trusted users or processes. Whoever controls the daemon's environment controls the hook. +For remote or cloud deployments, the daemon supports a custom auth hook for remotely consumable HTTP routes: `AGENT_DEVICE_HTTP_AUTH_HOOK` names a module path that is dynamically imported (with `AGENT_DEVICE_HTTP_AUTH_EXPORT` selecting the export). The host-local `/admin/human-control/*` route uses the daemon token instead. The hook runs with the daemon's full privileges, so treat it as part of your trusted computing base: point it only at a read-only path you control, never at a location writable by less-trusted users or processes. Whoever controls the daemon's environment controls the hook. + +Lease-owner human-control RPCs pass normal authentication and tenant/lease admission. They can +target only the admitted lease's device and cannot alter a host administrator's hold. Host +administration is a separate capability: the daemon accepts `/admin/human-control/*` only on its +loopback listener with the local daemon token, and `agent-device proxy` does not forward `/admin/*`. +Holds and leases are in-memory; neither survives daemon restart. If a hook is configured and its result does not attest a `tenantId`, the daemon refuses the request (401) outright — it never falls back to a tenant the client declares itself (RPC body `meta.tenantId` or `flags.tenant`, or the `x-agent-device-tenant` header on the upload/artifact-download/diagnostics routes), and it never admits the request unscoped either: a shared token must not let one caller claim another tenant's identity, nor read a tenant-owned session or artifact by simply declaring none. A hook must attest `tenantId` on every request it wants admitted; a deployment with no hook configured is unaffected.